From 1dae5a66b1f8ee780f16ecad92a4e8cbe0fc8f1b Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Wed, 11 Nov 2026 22:32:05 +0100 Subject: [PATCH] Add JSON_RECURSION_DEPTH_LIMIT to set how deep value walks recurse (#6081) Signed-off-by: Niels Lohmann --- .github/workflows/ubuntu.yml | 2 +- cmake/ci.cmake | 21 + docs/mkdocs/docs/api/macros/index.md | 2 + .../docs/api/macros/json_no_thread_local.md | 1 + .../api/macros/json_recursion_depth_limit.md | 307 +++++++++ docs/mkdocs/docs/community/assurance_case.md | 7 +- .../examples/json_recursion_depth_limit.cpp | 29 + .../json_recursion_depth_limit.output | 4 + docs/mkdocs/docs/features/macros.md | 10 + .../docs/features/parsing/untrusted_input.md | 30 +- docs/mkdocs/mkdocs.yml | 1 + include/nlohmann/detail/macro_scope.hpp | 6 + .../nlohmann/detail/recursion_depth_limit.hpp | 81 ++- include/nlohmann/json.hpp | 37 +- single_include/nlohmann/json.hpp | 124 +++- tests/CMakeLists.txt | 14 + tests/src/unit-allocator.cpp | 26 +- tests/src/unit-comparison.cpp | 20 +- tests/src/unit-custom-array-type.cpp | 8 +- tests/src/unit-custom-base-class.cpp | 4 +- tests/src/unit-diagnostic-positions.cpp | 24 +- tests/src/unit-diagnostics.cpp | 4 +- tests/src/unit-json_patch.cpp | 7 +- tests/src/unit-large_json_binary_writers.cpp | 29 +- tests/src/unit-large_json_conversions.cpp | 4 +- tests/src/unit-merge_diff.cpp | 5 +- tests/src/unit-merge_patch.cpp | 4 +- tests/src/unit-modifiers.cpp | 4 +- tests/src/unit-recursion_depth_limit.cpp | 614 ++++++++++++++++++ tests/src/unit-regression1.cpp | 6 +- 30 files changed, 1340 insertions(+), 95 deletions(-) create mode 100644 docs/mkdocs/docs/api/macros/json_recursion_depth_limit.md create mode 100644 docs/mkdocs/docs/examples/json_recursion_depth_limit.cpp create mode 100644 docs/mkdocs/docs/examples/json_recursion_depth_limit.output create mode 100644 tests/src/unit-recursion_depth_limit.cpp diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index e0cd83af9..0f6956254 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -153,7 +153,7 @@ jobs: container: ubuntu:24.04 strategy: matrix: - target: [ci_cmake_flags, ci_test_diagnostics, ci_test_diagnostic_positions, ci_test_noexceptions, ci_test_noimplicitconversions, ci_test_legacycomparison, ci_test_noglobaludls, ci_test_disableenumserialization, ci_test_disabletuplereferenceconversion, ci_test_skiplibraryversioncheck, ci_test_simdutf, ci_test_strict_nul_handling, ci_test_delete_deprecated_functions, ci_test_no_thread_local] + target: [ci_cmake_flags, ci_test_diagnostics, ci_test_diagnostic_positions, ci_test_noexceptions, ci_test_noimplicitconversions, ci_test_legacycomparison, ci_test_noglobaludls, ci_test_disableenumserialization, ci_test_disabletuplereferenceconversion, ci_test_skiplibraryversioncheck, ci_test_simdutf, ci_test_strict_nul_handling, ci_test_delete_deprecated_functions, ci_test_no_thread_local, ci_test_recursion_depth_limit_0] steps: - name: Install build-essential run: apt-get update ; apt-get install -y build-essential unzip wget git diff --git a/cmake/ci.cmake b/cmake/ci.cmake index b41ab8b62..7cc5adebe 100644 --- a/cmake/ci.cmake +++ b/cmake/ci.cmake @@ -342,6 +342,27 @@ add_custom_target(ci_test_no_thread_local COMMENT "Compile and test without thread-local storage" ) +############################################################################### +# Change the number of levels the walks over a value recurse into. +############################################################################### + +# JSON_RECURSION_DEPTH_LIMIT is 128 by default, which is what the whole test +# suite otherwise runs with. 0 sends every walk to its explicit stack at once, +# so this target runs the whole suite through the walks that the default only +# uses for values nested more deeply than 128 levels: results must not depend on +# the limit. The largest limit, 254, is covered by a build of +# tests/src/unit-recursion_depth_limit.cpp (see tests/CMakeLists.txt). +add_custom_target(ci_test_recursion_depth_limit_0 + COMMAND ${CMAKE_COMMAND} + -DCMAKE_BUILD_TYPE=Debug -GNinja + -DJSON_BuildTests=ON -DJSON_FastTests=ON + -DCMAKE_CXX_FLAGS=-DJSON_RECURSION_DEPTH_LIMIT=0 + -S${PROJECT_SOURCE_DIR} -B${PROJECT_BINARY_DIR}/build_recursion_depth_limit_0 + COMMAND ${CMAKE_COMMAND} --build ${PROJECT_BINARY_DIR}/build_recursion_depth_limit_0 + COMMAND cd ${PROJECT_BINARY_DIR}/build_recursion_depth_limit_0 && ${CMAKE_CTEST_COMMAND} --parallel ${N} --output-on-failure + COMMENT "Compile and test with JSON_RECURSION_DEPTH_LIMIT=0" +) + ############################################################################### # Coverage. ############################################################################### diff --git a/docs/mkdocs/docs/api/macros/index.md b/docs/mkdocs/docs/api/macros/index.md index 2ddf6f245..0392ed448 100644 --- a/docs/mkdocs/docs/api/macros/index.md +++ b/docs/mkdocs/docs/api/macros/index.md @@ -40,6 +40,8 @@ header. See also the [macro overview page](../../features/macros.md). - [**JSON_NO_IO**](json_no_io.md) - switch off functions relying on certain C++ I/O headers - [**JSON_NO_SIMD**](json_no_simd.md) - use only portable code instead of vector instructions (SSE2, SSSE3, NEON) - [**JSON_NO_THREAD_LOCAL**](json_no_thread_local.md) - switch off the use of `thread_local` storage +- [**JSON_RECURSION_DEPTH_LIMIT**](json_recursion_depth_limit.md) - set how many nesting levels the operations on a value + recurse into on the call stack (default 128) - [**JSON_SKIP_UNSUPPORTED_COMPILER_CHECK**](json_skip_unsupported_compiler_check.md) - do not warn about unsupported compilers - [**JSON_USE_GLOBAL_UDLS**](json_use_global_udls.md) - place user-defined string literals (UDLs) into the global namespace - [**JSON_USE_SIMDUTF**](json_use_simdutf.md) - use the simdutf library to accelerate UTF-8 validation diff --git a/docs/mkdocs/docs/api/macros/json_no_thread_local.md b/docs/mkdocs/docs/api/macros/json_no_thread_local.md index 400d8fab7..c842ef047 100644 --- a/docs/mkdocs/docs/api/macros/json_no_thread_local.md +++ b/docs/mkdocs/docs/api/macros/json_no_thread_local.md @@ -45,6 +45,7 @@ Copying and comparing fall back to working without the call stack there, as they ## See also +- [`JSON_RECURSION_DEPTH_LIMIT`](json_recursion_depth_limit.md) - set how many nesting levels the operations recurse into - [FAQ: Thread safety](../../home/faq.md#thread-safety) - thread-safety guarantees of the library - [Supported Macros](../../features/macros.md) - guide to all supported macros diff --git a/docs/mkdocs/docs/api/macros/json_recursion_depth_limit.md b/docs/mkdocs/docs/api/macros/json_recursion_depth_limit.md new file mode 100644 index 000000000..0d7dc66e3 --- /dev/null +++ b/docs/mkdocs/docs/api/macros/json_recursion_depth_limit.md @@ -0,0 +1,307 @@ +# JSON_RECURSION_DEPTH_LIMIT + +```cpp +#define JSON_RECURSION_DEPTH_LIMIT /* value */ +``` + +This macro sets how many nesting levels the operations that walk a value recurse into on the call stack. Beyond that, +they continue with an explicit stack on the heap. The value is an integer constant from `0` to `254`; the default is +`128`. + +The operations are copying a value, comparing two values, serializing with [`dump`](../basic_json/dump.md) or +[`operator<<`](../operator_ltlt.md), hashing with `std::hash`, [`merge_patch`](../basic_json/merge_patch.md), +[`update`](../basic_json/update.md) with `merge_objects`, [`diff`](../basic_json/diff.md), +[`merge_diff`](../basic_json/merge_diff.md), converting a value to another specialization of `basic_json`, destroying a +value, and the binary writers [`to_cbor`](../basic_json/to_cbor.md), +[`to_msgpack`](../basic_json/to_msgpack.md), [`to_ubjson`](../basic_json/to_ubjson.md), +[`to_bjdata`](../basic_json/to_bjdata.md), and [`to_bon8`](../basic_json/to_bon8.md). They share this one limit; none +has +a limit of its own. Recursing is the fastest way to walk a value, but a value nested deeply enough would exhaust the +call +stack, so the library recurses only as far as the limit and finishes the levels below with an explicit stack. This is +what makes all of them safe for untrusted input +(see [Nesting depth](../../features/parsing/untrusted_input.md#nesting-depth)). Parsing, the binary readers, +[`flatten`](../basic_json/flatten.md), [`unflatten`](../basic_json/unflatten.md), and +[`to_bson`](../basic_json/to_bson.md) always use an explicit stack and do not depend on the macro. The lockstep +comparison of two [`json_view`](../basic_json_view/index.md) documents has a fixed bound of its own, 48 levels, +and does not depend on it either. + +The call stack that these operations need grows with the limit: roughly *a few kilobytes plus a cost per level times the +limit*, where the cost per level is up to about 2.3 KiB in an unoptimized and up to about 0.5 KiB in an optimized build +(see [Stack needed](#stack-needed)). The default of `128` levels is tuned for threads with ordinary stacks of a few +hundred kilobytes or more. Set a lower limit if your threads have small stacks (RTOS tasks, fibers, threads of the musl +C +library, which has a default stack of 128 KiB), or a higher limit to trade stack for speed on deeply nested values. + +The macro changes only how much stack is used and how fast the operations are. It never changes a result: every +operation returns the same value for every setting of the limit, for values nested less deeply than the limit as well as +for values nested more deeply. + +## Default definition + +The default value is `128`. + +```cpp +#define JSON_RECURSION_DEPTH_LIMIT 128 +``` + +When the macro is not defined, the library defines it to its default value. + +## Notes + +!!! note "Allowed values" + + The value must be an integer constant expression from `0` to `254`; any other value (a negative number, a number + above `254`, a floating-point number such as `254.0`, or `true` or `false`) is rejected at compile time with an + error message that names the macro. A character of any character type (`'a'`, `L'a'`, `u8'a'`, ...) is rejected, + too, although it is an integer. + + - `0` is allowed, and means that every operation uses its explicit stack from the first level on and no operation + recurses over the nesting of a value at all. That needs a constant amount of call stack (at most 16 KiB in the + measurements below, for a value nested 20,000 levels deep) and is the setting for tiny stacks, at the price of + speed (see [Speed](#speed)). + - `254` is the largest allowed value. Copying a value and comparing two values (with `<=>`, or with the operators on + objects and arrays of a custom type) cannot pass the depth down as an argument, because the signatures of the + copy constructor and the operators are fixed. They count the levels they have descended into in a variable of type + `#!cpp std::uint8_t` that exists once per thread (see [`JSON_NO_THREAD_LOCAL`](json_no_thread_local.md)). Only + the levels that are descended into are counted, so the count never exceeds the limit, not even when user code (a + copy constructor of a custom base class, say) copies a value while a copy is going on. A byte holds `255` at + most, and the limit stays below that, so that the count cannot wrap around. (More levels would hardly be + useful: `254` levels need up to 576 KiB of stack in an unoptimized build already, twice as much as the + default.) + +### Stack needed + +The tables show the smallest stack that an operation needs, in KiB, at several limits, on a value that is nested +more deeply than the limit (limit + 10 levels). Each cell shows the result for Clang (Apple clang 21) and for +GCC 16, separated by a slash; both measured on macOS with arm64, compiling in C++20 mode. The operations run in a +thread whose stack has the size given, and are not given any stack that they do not need: the value is built, and +the result destroyed, outside of that thread. Stacks can only be set in steps of 16 KiB (the page size), so `16` +means "16 KiB or less". A row stands for all operations listed; the table shows the largest value among them. + +Unoptimized (`-O0`): + +| Operation | limit 16 | limit 32 | limit 64 | limit 128 | limit 254 | +|---|---:|---:|---:|---:|---:| +| `dump()` and `dump(indent)`, `operator<<` | 48 / 32 | 80 / 64 | 144 / 128 | 288 / 256 | 576 / 512 | +| copy construction and assignment | 32 / 16 | 48 / 32 | 112 / 80 | 224 / 144 | 432 / 304 | +| `==` | 16 / 16 | 16 / 16 | 16 / 16 | 48 / 32 | 96 / 48 | +| `<=>` (C++20) | 16 / 16 | 16 / 16 | 32 / 32 | 64 / 64 | 144 / 128 | +| `std::hash` | 16 / 16 | 16 / 16 | 32 / 32 | 80 / 64 | 160 / 128 | +| `merge_patch`, `update(j, true)`, `patch` | 32 / 16 | 48 / 32 | 112 / 80 | 224 / 144 | 432 / 304 | +| `diff` | 16 / 16 | 16 / 16 | 32 / 32 | 64 / 48 | 144 / 96 | +| `merge_diff` | 16 / 16 | 48 / 32 | 80 / 48 | 176 / 112 | 352 / 224 | +| `to_cbor`, `to_msgpack` | 16 / 16 | 32 / 16 | 64 / 32 | 112 / 80 | 224 / 160 | +| `to_ubjson`, `to_bjdata` | 16 / 16 | 16 / 16 | 48 / 32 | 96 / 64 | 176 / 112 | +| `to_bon8` | 16 / 16 | 16 / 16 | 16 / 16 | 32 / 16 | 64 / 48 | +| conversion to another `basic_json` type | 32 / 16 | 64 / 32 | 112 / 80 | 224 / 144 | 448 / 304 | +| destruction | 16 / 16 | 16 / 16 | 16 / 16 | 16 / 16 | 32 / 32 | + +Optimized (`-O2`): + +| Operation | limit 16 | limit 32 | limit 64 | limit 128 | limit 254 | +|---|---:|---:|---:|---:|---:| +| `dump()` and `dump(indent)`, `operator<<` | 16 / 16 | 16 / 16 | 16 / 16 | 16 / 16 | 32 / 32 | +| copy construction and assignment | 16 / 16 | 16 / 16 | 16 / 16 | 32 / 32 | 64 / 48 | +| `==` | 16 / 16 | 16 / 16 | 16 / 16 | 16 / 16 | 16 / 16 | +| `<=>` (C++20) | 16 / 16 | 16 / 16 | 16 / 16 | 16 / 16 | 48 / 16 | +| `std::hash` | 16 / 16 | 16 / 16 | 16 / 16 | 32 / 48 | 80 / 80 | +| `merge_patch`, `update(j, true)`, `patch` | 16 / 16 | 16 / 16 | 16 / 16 | 32 / 32 | 64 / 48 | +| `diff` | 16 / 16 | 16 / 16 | 16 / 32 | 48 / 48 | 96 / 112 | +| `merge_diff` | 16 / 16 | 16 / 16 | 16 / 16 | 48 / 32 | 96 / 64 | +| `to_cbor`, `to_msgpack` | 16 / 16 | 16 / 16 | 16 / 16 | 16 / 16 | 16 / 16 | +| `to_ubjson`, `to_bjdata` | 16 / 16 | 16 / 16 | 16 / 16 | 32 / 32 | 64 / 64 | +| `to_bon8` | 16 / 16 | 16 / 16 | 16 / 16 | 16 / 32 | 32 / 64 | +| conversion to another `basic_json` type | 16 / 16 | 16 / 16 | 16 / 16 | 32 / 48 | 64 / 96 | +| destruction | 16 / 16 | 16 / 16 | 16 / 16 | 16 / 16 | 16 / 16 | + +With a limit of `0`, every operation in the tables ran in a 16 KiB stack at `-O0` (Clang and GCC), on a value nested +20,000 levels deep. The operations that do not depend on the macro (`flatten`, `unflatten`, `to_bson`) needed +16 KiB or less at every limit. + +The rule of thumb is + +> stack needed ≈ a few KiB + cost per level × limit + +with a cost per level of up to 2.3 KiB (`dump` at `-O0` with Clang; 2.1 KiB with GCC), 1.7 KiB for copying and merging +(1.3 KiB with GCC), and 0.1 to 0.5 KiB for most other operations at `-O0`; and 0.5 KiB or less at `-O2`. The numbers +come from the difference between the limits `128` and `254` in the tables, so they are rough: other compilers, +other targets (the frames of x86-64 code are about as large), other standard libraries, `JSON_DIAGNOSTICS`, and +sanitizers change them. Measure with your own tool chain (see the example below) rather than relying on the tables. +The stack that your own code and the operating system need comes on top of these numbers. + +Which limit to choose, for the stack that you can spare for the library (the largest limit of those measured +for which the slowest operation needs at most that much): + +| Stack for the library | optimized (`-O2`) | unoptimized (`-O0`) | +|---|---:|---:| +| 32 KiB | 64 | 0 | +| 64 KiB | 128 | 16 | +| 128 KiB (the default stack of threads on musl) | 254 | 32 | +| 256 KiB | 254 | 64 | +| 512 KiB | 254 | 128 | +| 1 MiB or more | 254 | 254 | + +### Speed + +Operations are faster the more of a value they handle with plain recursion. The levels below the limit are handled with +an explicit stack, which has to be kept on the heap and costs more per level. The macro therefore trades speed against +stack. + +The table shows the time of copying a value (including destroying the copy), of `==` on two equal values, of `dump`, and +of `std::hash` for several limits, relative to the default of `128` (`1.00` means the same time; more is slower). It was +measured with Clang 21 at `-O2` in C++17 mode on macOS with arm64, as the best of seven runs of each setting (two +rounds, best of both). The documents are ordinary JSON files that are nested 7 (`canada.json`), 8 +(`citm_catalog.json`), 10 (`twitter.json`), and 2 levels deep (`jeopardy.json`); the other values are one array or object +with one member per level, nested 1,000 and 100,000 levels deep. The absolute time at the limit `128` is given in the +third column. Differences of up to about 0.1 are noise: a different limit changes the layout of the compiled code, and +timings of the same binary vary by about as much. The slow columns (limits 0 and 1) vary more: repeating `std::hash` on +`canada.json` gave 1.27 to 1.30 for limit 0 and 1.4 to 1.9 for limit 1, so read the cells of those two columns as +accurate to about 0.3. + +| Value | Operation | limit 128 (absolute) | limit 0 | limit 1 | limit 16 | limit 254 | +|---|---|---:|---:|---:|---:|---:| +| `canada.json` (2.1 MB) | copy | 3.5 ms | 1.80 | 1.80 | 0.99 | 1.00 | +| `canada.json` (2.1 MB) | `==` | 350.5 µs | 2.36 | 2.38 | 1.00 | 0.99 | +| `canada.json` (2.1 MB) | `dump` | 2.4 ms | 1.11 | 1.11 | 1.00 | 1.00 | +| `canada.json` (2.1 MB) | `std::hash` | 920.2 µs | 1.33 | 1.59 | 1.00 | 1.00 | +| `citm_catalog.json` (1.7 MB) | copy | 1.8 ms | 1.38 | 1.36 | 1.00 | 0.99 | +| `citm_catalog.json` (1.7 MB) | `==` | 275.5 µs | 1.31 | 1.31 | 0.99 | 1.00 | +| `citm_catalog.json` (1.7 MB) | `dump` | 314.6 µs | 1.24 | 1.23 | 1.00 | 1.00 | +| `citm_catalog.json` (1.7 MB) | `std::hash` | 309.1 µs | 1.77 | 1.42 | 1.06 | 1.05 | +| `twitter.json` (0.6 MB) | copy | 865.7 µs | 1.45 | 1.47 | 1.01 | 1.00 | +| `twitter.json` (0.6 MB) | `==` | 135.8 µs | 1.26 | 1.23 | 1.00 | 1.00 | +| `twitter.json` (0.6 MB) | `dump` | 224.6 µs | 1.12 | 1.12 | 1.01 | 1.00 | +| `twitter.json` (0.6 MB) | `std::hash` | 132.9 µs | 1.28 | 1.27 | 1.03 | 1.04 | +| `jeopardy.json` (53 MB) | copy | 111.1 ms | 1.22 | 1.25 | 1.00 | 1.00 | +| `jeopardy.json` (53 MB) | `==` | 23.5 ms | 1.29 | 1.53 | 1.07 | 1.00 | +| `jeopardy.json` (53 MB) | `dump` | 33.2 ms | 1.14 | 1.34 | 1.03 | 1.01 | +| `jeopardy.json` (53 MB) | `std::hash` | 19.9 ms | 1.17 | 1.52 | 1.05 | 1.00 | +| arrays, 1,000 levels | copy | 69.9 µs | 0.95 | 0.98 | 0.92 | 0.99 | +| arrays, 1,000 levels | `==` | 10.4 µs | 1.02 | 1.01 | 0.96 | 1.00 | +| arrays, 1,000 levels | `dump` | 9.6 µs | 0.97 | 0.96 | 0.97 | 1.06 | +| arrays, 1,000 levels | `std::hash` | 16.9 µs | 1.00 | 1.01 | 1.01 | 0.98 | +| objects, 1,000 levels | copy | 77.1 µs | 0.98 | 0.97 | 0.97 | 1.03 | +| objects, 1,000 levels | `==` | 12.0 µs | 0.96 | 0.95 | 0.95 | 1.07 | +| objects, 1,000 levels | `dump` | 12.6 µs | 0.97 | 0.95 | 0.98 | 1.01 | +| objects, 1,000 levels | `std::hash` | 17.3 µs | 0.99 | 1.01 | 0.98 | 1.00 | +| arrays, 100,000 levels | copy | 6.4 ms | 1.00 | 1.00 | 1.00 | 1.00 | +| arrays, 100,000 levels | `==` | 1.1 ms | 1.01 | 1.03 | 1.00 | 1.06 | +| arrays, 100,000 levels | `dump` | 917.9 µs | 1.01 | 1.01 | 1.02 | 1.01 | +| arrays, 100,000 levels | `std::hash` | 1.7 ms | 1.04 | 1.05 | 0.99 | 0.99 | +| objects, 100,000 levels | copy | 7.5 ms | 0.99 | 1.02 | 1.01 | 0.99 | +| objects, 100,000 levels | `==` | 1.3 ms | 1.02 | 1.09 | 1.08 | 0.96 | +| objects, 100,000 levels | `dump` | 1.2 ms | 1.06 | 0.99 | 1.01 | 0.99 | +| objects, 100,000 levels | `std::hash` | 1.8 ms | 1.00 | 1.00 | 1.00 | 1.00 | + +The results are as follows. + +- A limit of `16` or more does not change the speed of ordinary documents (differences are within noise), because they + are not nested more deeply than that anyway. A document nested more deeply than the limit is walked partly with the + explicit stack. +- A limit of `0` or `1` makes the operations on ordinary documents 1.1 to 2.4 times as slow, because even their first + levels are handled with the explicit stack. The slowdown is largest for `==` on a document of many small containers + (`canada.json`: 2.4 times) and for copying (up to 1.8 times). +- For values that are nested far more deeply than any limit (1,000 and 100,000 levels deep), the limit makes no + measurable difference: almost all of the levels are handled with the explicit stack for every setting. + Raising the limit gains nothing for them, and lowering it costs nothing. + +!!! note "Other macros" + + - With [`JSON_NO_THREAD_LOCAL`](json_no_thread_local.md) defined (or for Clang targeting MinGW, where the library + defines it by itself), copying and comparing never descend, so for them the limit does not matter. The other + operations still recurse as many levels as the limit says. The range of the limit is the same. + - The macro is not part of the [ABI tag](../../features/namespace.md), as `JSON_NO_THREAD_LOCAL` is not, and it is + not undefined at the end of `json.hpp`. + +!!! warning "Use the same value in every translation unit" + + All translation units of a program that include the library must define the same value. Otherwise, the program + violates the One Definition Rule: the function `nlohmann::detail::recursion_depth_limit()` and all the functions + that use it are defined differently in different translation units. Which limit a call uses then depends on which + functions the compiler inlined (an inlined copy uses the value of its own translation unit) and which out-of-line + copy of each remaining function the linker kept, which is not specified. A function compiled with one value can + call a function compiled with another, so a single operation can mix both limits. + + Mixing values cannot corrupt the byte per thread that counts the levels of copying and comparing: each limit + is at most `254`, and all of them stop counting at their own limit, so the count never exceeds `254`, whichever + translation unit's definition runs. The counter exists once per thread and `basic_json` specialization, and is + shared by all translation units. Nor can mixing values change any result, as no result depends on the limit. What + changes is how much stack is used and how fast the operations are. Nevertheless, avoid it: the program has undefined + behavior as far as the C++ standard is concerned. + +## Examples + +??? example "Example: lower the limit" + + The example nests an array 5,000 levels deep. With the limit set to 16, the library recurses over the first 16 + levels + and over the rest uses an explicit stack. The results are those of the default limit. + + ```cpp + --8<-- "examples/json_recursion_depth_limit.cpp" + ``` + + Output: + + ``` + --8<-- "examples/json_recursion_depth_limit.output" + ``` + +??? example "Example: find out how much stack an operation needs" + + The program runs `dump()` on a value nested `depth` levels deep in a thread with a stack of `kib` KiB. Compile it + with the options and the limit that you use, e.g. `-DJSON_RECURSION_DEPTH_LIMIT=64`, run it with `depth` set to the + limit plus 10 and decreasing values of `kib` (the stack size must be a multiple of the page size, often 4 or 16 + KiB), + and note the smallest stack with which it still finishes. That is how the tables above were measured. + + ```cpp + #include + #include + #include + #include + #include + #include + + static nlohmann::json value; + + static void* run(void*) + { + std::cout << "dump() ok, " << value.dump().size() << " bytes\n"; + return nullptr; + } + + int main(int argc, char** argv) + { + const int depth = std::atoi(argv[1]); + const std::size_t kib = static_cast(std::atoi(argv[2])); + + // built on the main thread, without recursion + value = 1; + for (int i = 0; i < depth; ++i) + { + nlohmann::json outer = nlohmann::json::array(); + outer.push_back(std::move(value)); + value = std::move(outer); + } + + pthread_attr_t attr; + pthread_attr_init(&attr); + pthread_attr_setstacksize(&attr, kib * 1024); + pthread_t thread; + pthread_create(&thread, &attr, run, nullptr); + pthread_join(thread, nullptr); + } + ``` + +## See also + +- [Nesting depth](../../features/parsing/untrusted_input.md#nesting-depth) - how the library handles deeply nested + values +- [`JSON_NO_THREAD_LOCAL`](json_no_thread_local.md) - switch off the use of `thread_local` storage +- [Supported Macros](../../features/macros.md) - guide to all supported macros + +## Version history + +- Added in version 3.13.0. diff --git a/docs/mkdocs/docs/community/assurance_case.md b/docs/mkdocs/docs/community/assurance_case.md index 02dc6d7d7..54c9a344b 100644 --- a/docs/mkdocs/docs/community/assurance_case.md +++ b/docs/mkdocs/docs/community/assurance_case.md @@ -53,12 +53,15 @@ flowchart LR only as bytes are actually read, arrays reserve at most a fixed number of elements up front, and sizes that no container can hold are rejected. - **Memory is owned by values.** Each `basic_json` value owns its content, and there is no manual memory management in - user code. The destructor does not recurse, so destroying a deeply nested value does not exhaust the stack. + user code. The destructor recurses only over a bounded number of nesting levels and then continues without the call + stack, so destroying a deeply nested value does not exhaust the stack. - **Bounded recursion.** The JSON parser and the binary readers keep their state in explicit stacks instead of recursing per nesting level. Operations that walk a value, such as [`dump`](../api/basic_json/dump.md), copying, comparison, hashing, [`merge_patch`](../api/basic_json/merge_patch.md), [`diff`](../api/basic_json/diff.md), [`flatten`](../api/basic_json/flatten.md), and the binary writers, recurse only up to a fixed depth and continue with - an explicit stack below it. Applications that process untrusted input can limit its nesting depth with a + an explicit stack below it. The depth is 128 levels by default and can be changed with + [`JSON_RECURSION_DEPTH_LIMIT`](../api/macros/json_recursion_depth_limit.md), for example to run on a small stack. + Applications that process untrusted input can limit its nesting depth with a [parser callback](../features/parsing/parser_callbacks.md). - **Invariants are checked.** The class invariant (for instance, that the pointer for the stored type is never null) is checked with runtime assertions throughout the test suite. diff --git a/docs/mkdocs/docs/examples/json_recursion_depth_limit.cpp b/docs/mkdocs/docs/examples/json_recursion_depth_limit.cpp new file mode 100644 index 000000000..7a2bfdfc3 --- /dev/null +++ b/docs/mkdocs/docs/examples/json_recursion_depth_limit.cpp @@ -0,0 +1,29 @@ +#include +#include +#include + +// walk over at most 16 nesting levels on the call stack, and over the rest with an explicit stack +#define JSON_RECURSION_DEPTH_LIMIT 16 +#include + +using json = nlohmann::json; + +int main() +{ + // an array nested 5,000 levels deep, built without recursion + json j = 1; + for (int i = 0; i < 5000; ++i) + { + json outer = json::array(); + outer.push_back(std::move(j)); + j = std::move(outer); + } + + // the results are those of any other limit + const json copy = j; + std::cout << std::boolalpha; + std::cout << "limit: " << JSON_RECURSION_DEPTH_LIMIT << '\n'; + std::cout << "copy equals original: " << (copy == j) << '\n'; + std::cout << "length of dump(): " << j.dump().size() << '\n'; + std::cout << "same hash: " << (std::hash {}(copy) == std::hash {}(j)) << '\n'; +} diff --git a/docs/mkdocs/docs/examples/json_recursion_depth_limit.output b/docs/mkdocs/docs/examples/json_recursion_depth_limit.output new file mode 100644 index 000000000..efb8175db --- /dev/null +++ b/docs/mkdocs/docs/examples/json_recursion_depth_limit.output @@ -0,0 +1,4 @@ +limit: 16 +copy equals original: true +length of dump(): 10001 +same hash: true diff --git a/docs/mkdocs/docs/features/macros.md b/docs/mkdocs/docs/features/macros.md index 33b2273b9..ace29de02 100644 --- a/docs/mkdocs/docs/features/macros.md +++ b/docs/mkdocs/docs/features/macros.md @@ -198,6 +198,16 @@ this is planned to become the default in version 4.0.0. See [full documentation of `JSON_PRECISE_STREAM_POSITION`](../api/macros/json_precise_stream_position.md). +## `JSON_RECURSION_DEPTH_LIMIT` + +Sets the number of nesting levels that the operations walking a value (copying, comparing, serializing, hashing, +merging, diffing, converting, destroying, and the binary writers) recurse into on the call stack before they continue +with an explicit stack on the heap. The default is `128`; allowed are the integers from `0` to `254`. Lower it for +threads with small stacks, raise it to trade stack for speed. Results are the same for every setting; the value must be +the same in all translation units of a program. + +See [full documentation of `JSON_RECURSION_DEPTH_LIMIT`](../api/macros/json_recursion_depth_limit.md). + ## `JSON_SKIP_LIBRARY_VERSION_CHECK` When defined, the library will not create a compiler warning when a different version of the library was already diff --git a/docs/mkdocs/docs/features/parsing/untrusted_input.md b/docs/mkdocs/docs/features/parsing/untrusted_input.md index 6492728d0..0353bdfbd 100644 --- a/docs/mkdocs/docs/features/parsing/untrusted_input.md +++ b/docs/mkdocs/docs/features/parsing/untrusted_input.md @@ -54,10 +54,32 @@ Once a value has been parsed, operations that walk it recursively — serializin [`dump`](../../api/basic_json/dump.md), hashing it, copying it, comparing two values with `#!cpp ==`, `#!cpp <`, or (in C++20) `#!cpp <=>`, merging with [`update`](../../api/basic_json/update.md), applying a [`merge_patch`](../../api/basic_json/merge_patch.md), creating a patch with [`diff`](../../api/basic_json/diff.md) or -[`merge_diff`](../../api/basic_json/merge_diff.md), [`flatten`](../../api/basic_json/flatten.md), and the binary -writers (`to_cbor`, `to_msgpack`, ...) — descend at most 128 levels on the call stack and continue -below that with an explicit stack instead, so none of them can exhaust the stack either, however deeply the value is -nested. Destroying a value (its destructor) never recurses at all, regardless of nesting depth, for the same reason. +[`merge_diff`](../../api/basic_json/merge_diff.md), converting it to another `basic_json` type, destroying it, and the +binary writers (`to_cbor`, `to_msgpack`, ...) — descend at most +[`JSON_RECURSION_DEPTH_LIMIT`](../../api/macros/json_recursion_depth_limit.md) levels on the call stack, 128 by default, +and continue below that with an explicit stack on the heap. So the call stack does not grow with the nesting depth of the +value, however deeply it is nested. ([`flatten`](../../api/basic_json/flatten.md), +[`unflatten`](../../api/basic_json/unflatten.md), and [`to_bson`](../../api/basic_json/to_bson.md) never use the call +stack for the nesting.) + +The call stack still needs room for the levels that are recursed into, and the default limit assumes a stack of ordinary +size. The stack that an operation needs is roughly proportional to the limit; with the default of 128 levels, the most +demanding operation (`dump` in an unoptimized build) needed up to 288 KiB, and an optimized build needed 48 KiB or less, +in measurements on macOS with arm64. The most demanding operation needed this much stack at the limits `0`, `32`, +`128`, and `254`: + +| Stack needed, in KiB | limit 0 | limit 32 | limit 128 (default) | limit 254 | +|---|---:|---:|---:|---:| +| unoptimized build (`-O0`) | 16 or less | 80 | 288 | 576 | +| optimized build (`-O2`) | 16 or less | 16 or less | 48 | 112 | + +A thread with a smaller stack (a task of a real-time operating system, a fiber, or a thread of the musl C library, whose +default stack is 128 KiB) can still overflow its stack on deeply nested input, which is why the limit can be lowered: +define `JSON_RECURSION_DEPTH_LIMIT` to a smaller value, down to `0`, before including the library. Such a limit changes +only the stack that is needed and the speed (values nested more deeply than the limit are walked more slowly), never a +result. See the [documentation of the macro](../../api/macros/json_recursion_depth_limit.md) for the complete tables, +the speed, and the rules for choosing a value. Destroying a value (its destructor) recurses over the same number of +levels and then continues without the call stack, and needs 32 KiB or less at every limit. ## Input size diff --git a/docs/mkdocs/mkdocs.yml b/docs/mkdocs/mkdocs.yml index 4c587ad9d..44e06c4ff 100644 --- a/docs/mkdocs/mkdocs.yml +++ b/docs/mkdocs/mkdocs.yml @@ -454,6 +454,7 @@ nav: - 'JSON_NO_SIMD': api/macros/json_no_simd.md - 'JSON_NO_THREAD_LOCAL': api/macros/json_no_thread_local.md - 'JSON_PRECISE_STREAM_POSITION': api/macros/json_precise_stream_position.md + - 'JSON_RECURSION_DEPTH_LIMIT': api/macros/json_recursion_depth_limit.md - 'JSON_SKIP_LIBRARY_VERSION_CHECK': api/macros/json_skip_library_version_check.md - 'JSON_SKIP_UNSUPPORTED_COMPILER_CHECK': api/macros/json_skip_unsupported_compiler_check.md - 'JSON_STRICT_BINARY_UTF8': api/macros/json_strict_binary_utf8.md diff --git a/include/nlohmann/detail/macro_scope.hpp b/include/nlohmann/detail/macro_scope.hpp index 208c1c3ce..76d191128 100644 --- a/include/nlohmann/detail/macro_scope.hpp +++ b/include/nlohmann/detail/macro_scope.hpp @@ -224,6 +224,12 @@ #define JSON_NO_THREAD_LOCAL 1 #endif +// JSON_RECURSION_DEPTH_LIMIT (the number of nesting levels the walks over a +// value recurse into before they continue with an explicit stack; 128 by +// default, 0 to 254 allowed) is defined and checked where it is used, in +// detail/recursion_depth_limit.hpp. Like JSON_NO_THREAD_LOCAL it is neither part +// of the ABI tag nor undefined at the end of json.hpp. + // allow disabling exceptions #if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && !defined(JSON_NOEXCEPTION) #define JSON_THROW(exception) throw exception diff --git a/include/nlohmann/detail/recursion_depth_limit.hpp b/include/nlohmann/detail/recursion_depth_limit.hpp index 6553a4d05..dced5d73c 100644 --- a/include/nlohmann/detail/recursion_depth_limit.hpp +++ b/include/nlohmann/detail/recursion_depth_limit.hpp @@ -9,9 +9,22 @@ #pragma once #include // size_t +#include // false_type, is_integral, remove_cv, true_type #include +// JSON_RECURSION_DEPTH_LIMIT: the number of nesting levels the operations that +// walk a value recurse into on the call stack before they continue with an +// explicit stack (see detail::recursion_depth_limit() below). +// +// May be defined by the user before the first include of the library; it has +// to be the same in every translation unit of a program (it is not part of the +// ABI tag, see https://json.nlohmann.me/api/macros/json_recursion_depth_limit/). +// Allowed: an integer constant expression from 0 to 254. The default is 128. +#ifndef JSON_RECURSION_DEPTH_LIMIT + #define JSON_RECURSION_DEPTH_LIMIT 128 +#endif + NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail { @@ -20,21 +33,71 @@ namespace detail @brief the number of nesting levels an operation recurses into Operations that walk a value (copying, comparing, serializing, hashing, merging, -...) recurse once -per nesting level, which is fastest, but a value nested deeply enough would -exhaust the call stack. So they recurse only this many levels deep and finish -whatever lies below with an explicit stack. All of them share this limit. +diffing, converting, destroying, and writing the binary formats CBOR, +MessagePack, UBJSON, BJData, and BON8) recurse once per nesting level, which is +fastest, but a value nested deeply enough would exhaust the call stack. So they +recurse only this many levels deep and finish whatever lies below with an +explicit stack that lives on the heap. All of them share this limit and none has +a constant of its own. It is the value of the macro JSON_RECURSION_DEPTH_LIMIT, +128 by default. (Parsing, the binary readers, flatten(), unflatten(), and +to_bson() always use an explicit stack and do not read it, and the lockstep +comparison of json_view documents has a fixed bound of its own, 48 levels.) -Most of them pass the depth down as an argument. The copy constructor and the -comparison operators cannot, as their signatures are fixed, so they count it -in basic_json::nesting_depth() instead, a byte per thread; the limit must -therefore stay below 255. +Most of the operations pass the depth down as an argument. The copy constructor +and the comparison operators cannot, as their signatures are fixed, so they +count it in basic_json::nesting_depth() instead, a byte per thread. Only the +levels that are descended into are counted, so the count never exceeds the limit. +The limit must fit in the byte with some headroom, so that the count cannot wrap +around: it must be below 255, and 254 is the largest value allowed. (More levels +would hardly be useful: 254 levels need up to 576 KiB of stack in an unoptimized +build already, twice the stack of the default.) +The smallest value is 0: every operation then uses its explicit stack from the +first level on and no operation recurses over the nesting of a value at all. +That is slower, but needs a constant amount of call stack. It works because +every use of the limit compares a depth with it (`depth >= limit`, or `<` for +the opposite), which holds for 0 as for any other value; nothing computes +`limit - 1` or `limit + 1`, so nothing wraps around. Each operation's explicit +stack handles any nesting level, including the first, so none of them relies on +having recursed first. + +The call stack that an operation needs is roughly a few KiB plus a cost per level +times the limit. At the default of 128 levels, an unoptimized build needs up to +about 290 KiB (`dump()` with Clang at -O0: about 2.3 KiB per level) and an +optimized build up to about 48 KiB; with 254 levels it is up to 576 KiB and +112 KiB, and with 0 it is 16 KiB or less. Lower the limit for threads with small +stacks (RTOS tasks, fibers, musl threads), raise it to trade stack for speed on +deeply nested values. All of this, with the measurements, is in the +documentation of the macro. + +@sa https://json.nlohmann.me/api/macros/json_recursion_depth_limit/ @sa https://github.com/nlohmann/json/issues/5387 */ +/// the type of the value of JSON_RECURSION_DEPTH_LIMIT is bool or a character type +template struct recursion_depth_limit_is_char_or_bool : std::false_type {}; +template<> struct recursion_depth_limit_is_char_or_bool : std::true_type {}; +template<> struct recursion_depth_limit_is_char_or_bool : std::true_type {}; +template<> struct recursion_depth_limit_is_char_or_bool : std::true_type {}; +template<> struct recursion_depth_limit_is_char_or_bool : std::true_type {}; +template<> struct recursion_depth_limit_is_char_or_bool : std::true_type {}; +template<> struct recursion_depth_limit_is_char_or_bool : std::true_type {}; +template<> struct recursion_depth_limit_is_char_or_bool : std::true_type {}; +#if defined(__cpp_char8_t) +template<> struct recursion_depth_limit_is_char_or_bool : std::true_type {}; +#endif + constexpr std::size_t recursion_depth_limit() noexcept { - return 128; + // an integer: not a floating-point number, whose range check would pass for 1.5 or 254.0 + // (bool and the character types are integer types, too, but true, false or 'a' are + // no way to say a limit; JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE refuses them as well) + static_assert(std::is_integral::value + && !recursion_depth_limit_is_char_or_bool::type>::value, + "JSON_RECURSION_DEPTH_LIMIT must be an integer constant (not bool or a character type)"); + static_assert(JSON_RECURSION_DEPTH_LIMIT >= 0 && JSON_RECURSION_DEPTH_LIMIT <= 254, + "JSON_RECURSION_DEPTH_LIMIT must be between 0 and 254: the nesting depth is counted in a byte per thread " + "and may exceed the limit by one (see https://json.nlohmann.me/api/macros/json_recursion_depth_limit/)"); + return static_cast(JSON_RECURSION_DEPTH_LIMIT); } } // namespace detail diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index bd06811d9..ea5977460 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -1256,8 +1256,10 @@ public: } #ifndef JSON_NO_THREAD_LOCAL - // nesting_depth() is a byte and may exceed the limit by one level - static_assert(detail::recursion_depth_limit() < 255, "the nesting depth count must fit in a byte"); + // nesting_depth() is a byte that never exceeds the limit (JSON_RECURSION_DEPTH_LIMIT), + // which is at most 254: the largest value stays one below the byte's maximum, so + // that the count cannot wrap around to 0 and let a descent go on without a bound. + // recursion_depth_limit() checks this range with a static_assert. /*! @brief how many levels the operation going on in this thread has descended into @@ -1268,8 +1270,15 @@ public: user code nests them anyway, sharing the count only ends a descent sooner than it had to, which costs a little speed and is never wrong. - A byte is enough: the count never exceeds the limit by more than the single - level that notices the limit has been reached. + A byte is enough: only the levels that are descended into are counted + (@ref nesting_depth_guard), so the count never exceeds the limit, and the + limit is at most 254. + + The count is per thread and not per value or per library configuration: all + translation units of a program share it, even if they were compiled with + different values of JSON_RECURSION_DEPTH_LIMIT (which is not supported, see + the documentation of the macro); each of them then compares the shared count + with its own limit. */ static std::uint8_t& nesting_depth() noexcept { @@ -1327,14 +1336,25 @@ public: #endif { #ifndef JSON_NO_THREAD_LOCAL - ++nesting_depth(); + // Only a level that is entered is counted, so that the count never + // exceeds the limit. A level that is not entered is finished without + // descending, and whatever runs while it is (a copy made by a custom + // base class, say) sees the same count as the level itself did and + // is not entered either. + if (m_okay) + { + ++nesting_depth(); + } #endif } ~nesting_depth_guard() { #ifndef JSON_NO_THREAD_LOCAL - --nesting_depth(); + if (m_okay) + { + --nesting_depth(); + } #endif } @@ -5246,7 +5266,10 @@ public: */ void update_members(const iterator& first, const iterator& last, const bool merge_objects, const std::size_t depth) { - if (JSON_HEDLEY_UNLIKELY(depth >= detail::recursion_depth_limit())) + // without merge_objects the loop below never descends, so there is no + // call stack to save (and the explicit stack always merges); the + // distinction matters for a limit of 0, which reaches this at depth 0 + if (JSON_HEDLEY_UNLIKELY(merge_objects && depth >= detail::recursion_depth_limit())) { update_members_iteratively(first, last); return; diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 6fa5bbbb6..112441bf1 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -2597,6 +2597,12 @@ JSON_HEDLEY_DIAGNOSTIC_POP #define JSON_NO_THREAD_LOCAL 1 #endif +// JSON_RECURSION_DEPTH_LIMIT (the number of nesting levels the walks over a +// value recurse into before they continue with an explicit stack; 128 by +// default, 0 to 254 allowed) is defined and checked where it is used, in +// detail/recursion_depth_limit.hpp. Like JSON_NO_THREAD_LOCAL it is neither part +// of the ABI tag nor undefined at the end of json.hpp. + // allow disabling exceptions #if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && !defined(JSON_NOEXCEPTION) #define JSON_THROW(exception) throw exception @@ -8600,10 +8606,23 @@ NLOHMANN_JSON_NAMESPACE_END #include // size_t +#include // false_type, is_integral, remove_cv, true_type // #include +// JSON_RECURSION_DEPTH_LIMIT: the number of nesting levels the operations that +// walk a value recurse into on the call stack before they continue with an +// explicit stack (see detail::recursion_depth_limit() below). +// +// May be defined by the user before the first include of the library; it has +// to be the same in every translation unit of a program (it is not part of the +// ABI tag, see https://json.nlohmann.me/api/macros/json_recursion_depth_limit/). +// Allowed: an integer constant expression from 0 to 254. The default is 128. +#ifndef JSON_RECURSION_DEPTH_LIMIT + #define JSON_RECURSION_DEPTH_LIMIT 128 +#endif + NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail { @@ -8612,21 +8631,71 @@ namespace detail @brief the number of nesting levels an operation recurses into Operations that walk a value (copying, comparing, serializing, hashing, merging, -...) recurse once -per nesting level, which is fastest, but a value nested deeply enough would -exhaust the call stack. So they recurse only this many levels deep and finish -whatever lies below with an explicit stack. All of them share this limit. +diffing, converting, destroying, and writing the binary formats CBOR, +MessagePack, UBJSON, BJData, and BON8) recurse once per nesting level, which is +fastest, but a value nested deeply enough would exhaust the call stack. So they +recurse only this many levels deep and finish whatever lies below with an +explicit stack that lives on the heap. All of them share this limit and none has +a constant of its own. It is the value of the macro JSON_RECURSION_DEPTH_LIMIT, +128 by default. (Parsing, the binary readers, flatten(), unflatten(), and +to_bson() always use an explicit stack and do not read it, and the lockstep +comparison of json_view documents has a fixed bound of its own, 48 levels.) -Most of them pass the depth down as an argument. The copy constructor and the -comparison operators cannot, as their signatures are fixed, so they count it -in basic_json::nesting_depth() instead, a byte per thread; the limit must -therefore stay below 255. +Most of the operations pass the depth down as an argument. The copy constructor +and the comparison operators cannot, as their signatures are fixed, so they +count it in basic_json::nesting_depth() instead, a byte per thread. Only the +levels that are descended into are counted, so the count never exceeds the limit. +The limit must fit in the byte with some headroom, so that the count cannot wrap +around: it must be below 255, and 254 is the largest value allowed. (More levels +would hardly be useful: 254 levels need up to 576 KiB of stack in an unoptimized +build already, twice the stack of the default.) +The smallest value is 0: every operation then uses its explicit stack from the +first level on and no operation recurses over the nesting of a value at all. +That is slower, but needs a constant amount of call stack. It works because +every use of the limit compares a depth with it (`depth >= limit`, or `<` for +the opposite), which holds for 0 as for any other value; nothing computes +`limit - 1` or `limit + 1`, so nothing wraps around. Each operation's explicit +stack handles any nesting level, including the first, so none of them relies on +having recursed first. + +The call stack that an operation needs is roughly a few KiB plus a cost per level +times the limit. At the default of 128 levels, an unoptimized build needs up to +about 290 KiB (`dump()` with Clang at -O0: about 2.3 KiB per level) and an +optimized build up to about 48 KiB; with 254 levels it is up to 576 KiB and +112 KiB, and with 0 it is 16 KiB or less. Lower the limit for threads with small +stacks (RTOS tasks, fibers, musl threads), raise it to trade stack for speed on +deeply nested values. All of this, with the measurements, is in the +documentation of the macro. + +@sa https://json.nlohmann.me/api/macros/json_recursion_depth_limit/ @sa https://github.com/nlohmann/json/issues/5387 */ +/// the type of the value of JSON_RECURSION_DEPTH_LIMIT is bool or a character type +template struct recursion_depth_limit_is_char_or_bool : std::false_type {}; +template<> struct recursion_depth_limit_is_char_or_bool : std::true_type {}; +template<> struct recursion_depth_limit_is_char_or_bool : std::true_type {}; +template<> struct recursion_depth_limit_is_char_or_bool : std::true_type {}; +template<> struct recursion_depth_limit_is_char_or_bool : std::true_type {}; +template<> struct recursion_depth_limit_is_char_or_bool : std::true_type {}; +template<> struct recursion_depth_limit_is_char_or_bool : std::true_type {}; +template<> struct recursion_depth_limit_is_char_or_bool : std::true_type {}; +#if defined(__cpp_char8_t) +template<> struct recursion_depth_limit_is_char_or_bool : std::true_type {}; +#endif + constexpr std::size_t recursion_depth_limit() noexcept { - return 128; + // an integer: not a floating-point number, whose range check would pass for 1.5 or 254.0 + // (bool and the character types are integer types, too, but true, false or 'a' are + // no way to say a limit; JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE refuses them as well) + static_assert(std::is_integral::value + && !recursion_depth_limit_is_char_or_bool::type>::value, + "JSON_RECURSION_DEPTH_LIMIT must be an integer constant (not bool or a character type)"); + static_assert(JSON_RECURSION_DEPTH_LIMIT >= 0 && JSON_RECURSION_DEPTH_LIMIT <= 254, + "JSON_RECURSION_DEPTH_LIMIT must be between 0 and 254: the nesting depth is counted in a byte per thread " + "and may exceed the limit by one (see https://json.nlohmann.me/api/macros/json_recursion_depth_limit/)"); + return static_cast(JSON_RECURSION_DEPTH_LIMIT); } } // namespace detail @@ -40447,8 +40516,10 @@ public: } #ifndef JSON_NO_THREAD_LOCAL - // nesting_depth() is a byte and may exceed the limit by one level - static_assert(detail::recursion_depth_limit() < 255, "the nesting depth count must fit in a byte"); + // nesting_depth() is a byte that never exceeds the limit (JSON_RECURSION_DEPTH_LIMIT), + // which is at most 254: the largest value stays one below the byte's maximum, so + // that the count cannot wrap around to 0 and let a descent go on without a bound. + // recursion_depth_limit() checks this range with a static_assert. /*! @brief how many levels the operation going on in this thread has descended into @@ -40459,8 +40530,15 @@ public: user code nests them anyway, sharing the count only ends a descent sooner than it had to, which costs a little speed and is never wrong. - A byte is enough: the count never exceeds the limit by more than the single - level that notices the limit has been reached. + A byte is enough: only the levels that are descended into are counted + (@ref nesting_depth_guard), so the count never exceeds the limit, and the + limit is at most 254. + + The count is per thread and not per value or per library configuration: all + translation units of a program share it, even if they were compiled with + different values of JSON_RECURSION_DEPTH_LIMIT (which is not supported, see + the documentation of the macro); each of them then compares the shared count + with its own limit. */ static std::uint8_t& nesting_depth() noexcept { @@ -40518,14 +40596,25 @@ public: #endif { #ifndef JSON_NO_THREAD_LOCAL - ++nesting_depth(); + // Only a level that is entered is counted, so that the count never + // exceeds the limit. A level that is not entered is finished without + // descending, and whatever runs while it is (a copy made by a custom + // base class, say) sees the same count as the level itself did and + // is not entered either. + if (m_okay) + { + ++nesting_depth(); + } #endif } ~nesting_depth_guard() { #ifndef JSON_NO_THREAD_LOCAL - --nesting_depth(); + if (m_okay) + { + --nesting_depth(); + } #endif } @@ -44437,7 +44526,10 @@ public: */ void update_members(const iterator& first, const iterator& last, const bool merge_objects, const std::size_t depth) { - if (JSON_HEDLEY_UNLIKELY(depth >= detail::recursion_depth_limit())) + // without merge_objects the loop below never descends, so there is no + // call stack to save (and the explicit stack always merges); the + // distinction matters for a limit of 0, which reaches this at depth 0 + if (JSON_HEDLEY_UNLIKELY(merge_objects && depth >= detail::recursion_depth_limit())) { update_members_iteratively(first, last); return; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index f4e574fc1..e7d90cecd 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -355,6 +355,20 @@ json_test_add_test_for(src/unit-comparison.cpp MAIN test_main CXX_STANDARDS ${test_cxx_standards} ${test_force} ) +# test the walks over deeply nested values with the extreme settings of +# JSON_RECURSION_DEPTH_LIMIT: 0 (every walk uses its explicit stack from the +# first level on), 1, and 254 (the largest limit that fits the byte that counts +# the nesting of a copy) +foreach(recursion_depth_limit 0 1 254) + json_test_set_test_options(test-recursion_depth_limit_${recursion_depth_limit} + COMPILE_DEFINITIONS JSON_RECURSION_DEPTH_LIMIT=${recursion_depth_limit} + ) + json_test_add_test_for(src/unit-recursion_depth_limit.cpp + NAME test-recursion_depth_limit_${recursion_depth_limit} + MAIN test_main CXX_STANDARDS ${test_cxx_standards} ${test_force} + ) +endforeach() + # test the parser again with JSON_DIAGNOSTIC_POSITIONS enabled json_test_set_test_options(test-class_parser_diagnostic_positions COMPILE_DEFINITIONS JSON_DIAGNOSTIC_POSITIONS=1 diff --git a/tests/src/unit-allocator.cpp b/tests/src/unit-allocator.cpp index 9862e8c5e..73693347f 100644 --- a/tests/src/unit-allocator.cpp +++ b/tests/src/unit-allocator.cpp @@ -236,7 +236,8 @@ TEST_CASE("controlled bad_alloc") next_construct_fails = false; - // deeper than the 128 levels the copy constructor descends into + // deeper than the levels the copy constructor descends into + // (recursion_depth_limit(): 128 by default, 254 at most) const std::size_t depth = 300; my_json j = 1; @@ -300,8 +301,9 @@ struct nth_alloc_fails_allocator : std::allocator }; }; -// builds a value nested more than 128 levels deep - the bound the copy -// constructor descends into before it continues without the call stack - and +// builds a value nested more than recursion_depth_limit() (128 by default) +// levels deep - the bound the copy constructor descends into before it +// continues without the call stack - and // checks that a copy survives any single allocation of it failing: every // attempt either throws std::bad_alloc, without crashing or leaving the // source altered, or completes the copy @@ -312,9 +314,9 @@ void check_deep_copy_survives_failing_allocation(bool nest_objects) fail_at_alloc_call = -1; - // [[[ ... [1] ... ]]], or the same nesting with objects, 130 levels deep + // [[[ ... [1] ... ]]], or the same nesting with objects, two levels deeper than the limit BasicJsonType src = 1; - for (std::size_t i = 0; i < 130; ++i) + for (std::size_t i = 0; i < nlohmann::detail::recursion_depth_limit() + 2; ++i) { if (nest_objects) { @@ -440,8 +442,9 @@ TEST_CASE("deep copy uses the provided allocator") { using counting_json = nlohmann::json::with_allocator_t; - // deeper than the 128 levels the copy constructor descends into, so the - // innermost objects are copied by the iterative deep copy + // deeper than the levels the copy constructor descends into (128 by + // default, 254 at most), so the innermost objects are copied by the + // iterative deep copy counting_json j = 1; for (std::size_t i = 0; i < 300; ++i) { @@ -499,11 +502,12 @@ TEST_CASE("converting a deeply nested value from another specialization fails cl #if !(defined(_MSC_VER) && _MSC_VER < 1910 && defined(_ITERATOR_DEBUG_LEVEL) && _ITERATOR_DEBUG_LEVEL > 0) using countdown_json = nlohmann::json::with_allocator_t; - // deeper than the 128 levels the converting constructor descends into, so - // that failures land on both sides of the bound - or, built with - // JSON_NO_THREAD_LOCAL, all in the iterative conversion + // deeper than the levels the converting constructor descends into + // (recursion_depth_limit(), 128 by default), so that failures land on both + // sides of the bound - or, built with JSON_NO_THREAD_LOCAL, all in the + // iterative conversion json j = {1, "two", {{"three", 3}}}; - for (std::size_t i = 0; i < 150; ++i) + for (std::size_t i = 0; i < nlohmann::detail::recursion_depth_limit() + 22; ++i) { j = json{{"a", json::array({j, "sibling"})}}; } diff --git a/tests/src/unit-comparison.cpp b/tests/src/unit-comparison.cpp index 4ac43320b..bec6b136e 100644 --- a/tests/src/unit-comparison.cpp +++ b/tests/src/unit-comparison.cpp @@ -62,6 +62,15 @@ template<> struct StringMaker namespace { +// nesting depths on either side of detail::recursion_depth_limit(): none, the +// last one within the limit, the first one past it, and well past it (0, 127, +// 128, and 200 by default) +std::vector depths_around_recursion_limit() +{ + const std::size_t limit = nlohmann::detail::recursion_depth_limit(); + return {0, limit > 0 ? limit - 1 : 0, limit, limit + 72}; +} + // helper function to check std::less // see https://en.cppreference.com/w/cpp/utility/functional/less template > @@ -968,7 +977,7 @@ TEST_CASE("copying an object preserves its comparator's state") object["a"] = 3; REQUIRE(object.dump() == R"({"B":2,"a":3,"b":1})"); - for (const std::size_t depth : std::vector {0, 127, 128, 200}) + for (const std::size_t depth : depths_around_recursion_limit()) { CAPTURE(depth) @@ -1011,7 +1020,7 @@ TEST_CASE("equality of an object whose comparator treats different keys as equiv // sanity check: the object type's own comparison already disagrees CHECK_FALSE(a.get_ref() == b.get_ref()); - for (const std::size_t depth : std::vector {0, 127, 128, 200}) + for (const std::size_t depth : depths_around_recursion_limit()) { CAPTURE(depth) @@ -1135,9 +1144,10 @@ TEST_CASE("operator<=> of binary values with a different subtype does not depend return result; }; - // 127 levels stay within nesting_depth_limit() (128); 128 and 200 do not, - // and must still agree with the levels that do - for (const std::size_t depth : std::vector {0, 127, 128, 200}) + // one level less than the recursion depth limit (127 by default) stays within it; + // the limit itself and more (128 and 200) do not, and must still agree with the + // levels that do + for (const std::size_t depth : depths_around_recursion_limit()) { CAPTURE(depth) const json x = deep(a, depth); diff --git a/tests/src/unit-custom-array-type.cpp b/tests/src/unit-custom-array-type.cpp index 65285eb54..e5e3307b9 100644 --- a/tests/src/unit-custom-array-type.cpp +++ b/tests/src/unit-custom-array-type.cpp @@ -11,6 +11,7 @@ #include using nlohmann::json; +#include #include #include #include @@ -217,16 +218,17 @@ TEST_CASE("array type with std::deque nested beyond the recursion limit") // recursion_depth_limit() with an explicit stack of frames; with the // iterators of std::deque, these frames are copied and moved through // their own special members + const std::size_t depth = nlohmann::detail::recursion_depth_limit() + 72; // 200 by default deque_json j = 1; - for (int i = 0; i < 200; ++i) + for (std::size_t i = 0; i < depth; ++i) { deque_json outer = deque_json::array(); outer.push_back(std::move(j)); j = std::move(outer); } - // 200 arrays of one element each (0x81) around the integer 1 - std::vector expected(200, 0x81); + // `depth` arrays of one element each (0x81) around the integer 1 + std::vector expected(depth, 0x81); expected.push_back(0x01); CHECK(deque_json::to_cbor(j) == expected); } diff --git a/tests/src/unit-custom-base-class.cpp b/tests/src/unit-custom-base-class.cpp index 71698d041..c3033afe9 100644 --- a/tests/src/unit-custom-base-class.cpp +++ b/tests/src/unit-custom-base-class.cpp @@ -475,10 +475,10 @@ TEST_CASE("Regression test for issue #5674 - copy construction must not require SECTION("nested deeper than the copy constructor's descent bound") { - // beyond nesting_depth_limit() (128) levels, the copy constructor + // beyond recursion_depth_limit() (128 by default) levels, the copy constructor // copies without the call stack (copy_iteratively / copy_array_level), // which used to assign the base class of every element it created - const std::size_t depth = 300; + const std::size_t depth = nlohmann::detail::recursion_depth_limit() + 172; const json_with_const_base j = make_nested_array(depth); const json_with_const_base copy = j; // NOLINT(performance-unnecessary-copy-initialization) diff --git a/tests/src/unit-diagnostic-positions.cpp b/tests/src/unit-diagnostic-positions.cpp index 85284d9d4..8722a8787 100644 --- a/tests/src/unit-diagnostic-positions.cpp +++ b/tests/src/unit-diagnostic-positions.cpp @@ -128,16 +128,19 @@ TEST_CASE("Better diagnostics with positions") check_copy(depth, true); }; + // around detail::recursion_depth_limit() (127, 128, and 129 by default) + const std::size_t limit = nlohmann::detail::recursion_depth_limit(); + check_arrays(1); - check_arrays(127); - check_arrays(128); - check_arrays(129); + check_arrays(limit > 0 ? limit - 1 : 0); + check_arrays(limit); + check_arrays(limit + 1); check_arrays(300); check_objects(1); - check_objects(127); - check_objects(128); - check_objects(129); + check_objects(limit > 0 ? limit - 1 : 0); + check_objects(limit); + check_objects(limit + 1); check_objects(300); } @@ -186,10 +189,13 @@ TEST_CASE("Better diagnostics with positions") } }; + // around detail::recursion_depth_limit() (127, 128, and 129 by default) + const std::size_t limit = nlohmann::detail::recursion_depth_limit(); + check_conversion(1); - check_conversion(127); - check_conversion(128); - check_conversion(129); + check_conversion(limit > 0 ? limit - 1 : 0); + check_conversion(limit); + check_conversion(limit + 1); check_conversion(300); } diff --git a/tests/src/unit-diagnostics.cpp b/tests/src/unit-diagnostics.cpp index 2987d348c..801b041cd 100644 --- a/tests/src/unit-diagnostics.cpp +++ b/tests/src/unit-diagnostics.cpp @@ -515,9 +515,11 @@ TEST_CASE("Regression tests for extended diagnostics") // update(j, true) around its descent bound, where the nested vectors // grow while the objects are merged without recursing + const std::size_t limit = nlohmann::detail::recursion_depth_limit(); + // (the limit can be 0, where "one level less" is no level) for (const std::size_t depth : { - nlohmann::detail::recursion_depth_limit() - 1, nlohmann::detail::recursion_depth_limit(), nlohmann::detail::recursion_depth_limit() + 2 + limit > 0 ? limit - 1 : 0, limit, limit + 2 }) { ordered_json j = {{"z", {{"x", 1}}}}; diff --git a/tests/src/unit-json_patch.cpp b/tests/src/unit-json_patch.cpp index fead8a6d6..6b38cbcd6 100644 --- a/tests/src/unit-json_patch.cpp +++ b/tests/src/unit-json_patch.cpp @@ -1978,14 +1978,15 @@ TEST_CASE("JSON patch: diff of deeply nested values") SECTION("the diff reproduces the target at every depth") { // depths on either side of the nesting depth up to which diff() - // recurses (detail::recursion_depth_limit(), 128); not every depth up + // recurses (detail::recursion_depth_limit(), 128 by default); not every depth up // to 300, as the test would then time out under Valgrind std::vector depths; for (std::size_t depth = 0; depth <= 16; ++depth) { depths.push_back(depth); } - for (std::size_t depth = 120; depth <= 136; ++depth) + const std::size_t limit = nlohmann::detail::recursion_depth_limit(); + for (std::size_t depth = limit > 8 ? limit - 8 : 0; depth <= limit + 8; ++depth) { depths.push_back(depth); } @@ -2527,7 +2528,7 @@ TEST_CASE("JSON patch - diff of leaf values below the bound of the recursive des // the values at the innermost level are diffed without recursion: equal // leaves yield nothing, leaves of different types and different leaves // are replaced - const std::size_t depth = 200; + const std::size_t depth = nlohmann::detail::recursion_depth_limit() + 72; // 200 by default json source = {1, "text", 3, {{"x", 1}, {"y", 2}}, true, nullptr}; json target = {1, 2, 4, {{"x", 1}, {"y", 3}}, true, nullptr}; diff --git a/tests/src/unit-large_json_binary_writers.cpp b/tests/src/unit-large_json_binary_writers.cpp index 339c62135..8c8755e81 100644 --- a/tests/src/unit-large_json_binary_writers.cpp +++ b/tests/src/unit-large_json_binary_writers.cpp @@ -50,13 +50,18 @@ json nested_object(const std::size_t depth, json leaf) TEST_CASE("issue #5392 - binary writers on deeply nested values") { - // 200 is past the point where the writers stop recursing, and still - // shallow enough that from_* and operator== (which still recurse) are fine. - const json deep_array = nested_array(200, json(0)); - const json deep_object = nested_object(200, json("x")); - const json empty_array = nested_array(200, json::array()); - const json empty_object = nested_object(200, json::object()); - const json mixed = nested_object(80, nested_array(80, json(true))); + // The limit (128 by default) of the levels that the writers recurse into. + const std::size_t limit = nlohmann::detail::recursion_depth_limit(); + + // 72 levels more than the limit (200 by default) is past the point where + // the writers stop recursing, and still shallow enough that from_* (which + // does not recurse) and operator== (which does, up to the limit) are quick. + const json deep_array = nested_array(limit + 72, json(0)); + const json deep_object = nested_object(limit + 72, json("x")); + const json empty_array = nested_array(limit + 72, json::array()); + const json empty_object = nested_object(limit + 72, json::object()); + // 2 * (limit / 2 + 16) levels: 160 by default, and more than the limit for every limit + const json mixed = nested_object((limit / 2) + 16, nested_array((limit / 2) + 16, json(true))); SECTION("roundtrip past the recursion bound") { @@ -95,7 +100,7 @@ TEST_CASE("issue #5392 - binary writers on deeply nested values") SECTION("the two ways of writing a value meet at the bound") { - for (std::size_t depth = 120; depth <= 140; ++depth) + for (std::size_t depth = limit > 8 ? limit - 8 : 0; depth <= limit + 12; ++depth) { CAPTURE(depth) @@ -118,8 +123,8 @@ TEST_CASE("issue #5392 - binary writers on deeply nested values") const json ndarray = json({{"_ArrayType_", "uint8"}, {"_ArraySize_", {2, 3}}, {"_ArrayData_", {1, 2, 3, 4, 5, 6}}}); const json invalid = json({{"_ArrayType_", "nope"}, {"_ArraySize_", {1}}, {"_ArrayData_", {1}}}); - const json deep_ndarray = nested_array(140, ndarray); - const json deep_invalid = nested_array(140, invalid); + const json deep_ndarray = nested_array(limit + 12, ndarray); + const json deep_invalid = nested_array(limit + 12, invalid); CHECK(json::from_bjdata(json::to_bjdata(deep_ndarray)) == deep_ndarray); CHECK(json::from_bjdata(json::to_bjdata(deep_invalid)) == deep_invalid); @@ -131,10 +136,10 @@ TEST_CASE("issue #5392 - binary writers on deeply nested values") // nested one-element arrays around the recursion bound: the exact // bytes a writer produces do not depend on whether it stayed on the // call stack or moved to the heap one partway through + // (the limit can be 0, where "one level less" is no level) for (const std::size_t depth : { - nlohmann::detail::recursion_depth_limit() - 1, nlohmann::detail::recursion_depth_limit(), - nlohmann::detail::recursion_depth_limit() + 1, nlohmann::detail::recursion_depth_limit() + 2 + limit > 0 ? limit - 1 : 0, limit, limit + 1, limit + 2 }) { CAPTURE(depth) diff --git a/tests/src/unit-large_json_conversions.cpp b/tests/src/unit-large_json_conversions.cpp index 1fd30aa89..ba584df62 100644 --- a/tests/src/unit-large_json_conversions.cpp +++ b/tests/src/unit-large_json_conversions.cpp @@ -115,7 +115,7 @@ TEST_CASE("issue #5650 - stack overflow converting between specializations") // members that the two object types enumerate in different orders. const auto bury = [](nlohmann::ordered_json value) { - for (std::size_t i = 0; i < 200; ++i) + for (std::size_t i = 0; i < nlohmann::detail::recursion_depth_limit() + 72; ++i) { value = nlohmann::ordered_json::array({std::move(value)}); } @@ -124,7 +124,7 @@ TEST_CASE("issue #5650 - stack overflow converting between specializations") const auto dig = [](const json & value) { const json* current = &value; - for (std::size_t i = 0; i < 200; ++i) + for (std::size_t i = 0; i < nlohmann::detail::recursion_depth_limit() + 72; ++i) { current = ¤t->at(0); } diff --git a/tests/src/unit-merge_diff.cpp b/tests/src/unit-merge_diff.cpp index 77fa58768..6b3e67278 100644 --- a/tests/src/unit-merge_diff.cpp +++ b/tests/src/unit-merge_diff.cpp @@ -930,14 +930,15 @@ TEST_CASE("JSON Merge Patch diff of deeply nested values") SECTION("the patch is the same past the descent bound") { // depths on either side of the nesting depth up to which merge_diff() - // recurses (detail::recursion_depth_limit(), 128); not every depth up + // recurses (detail::recursion_depth_limit(), 128 by default); not every depth up // to 300, to keep the test fast under Valgrind std::vector depths; for (std::size_t depth = 0; depth <= 16; ++depth) { depths.push_back(depth); } - for (std::size_t depth = 120; depth <= 136; ++depth) + const std::size_t limit = nlohmann::detail::recursion_depth_limit(); + for (std::size_t depth = limit > 8 ? limit - 8 : 0; depth <= limit + 8; ++depth) { depths.push_back(depth); } diff --git a/tests/src/unit-merge_patch.cpp b/tests/src/unit-merge_patch.cpp index 3f5d79b3c..bb48cf070 100644 --- a/tests/src/unit-merge_patch.cpp +++ b/tests/src/unit-merge_patch.cpp @@ -302,7 +302,7 @@ TEST_CASE("JSON Merge Patch on deeply nested values") SECTION("patching past the descent bound gives the same result") { // every depth on either side of where the iterative version takes - // over (detail::recursion_depth_limit(), 128) + // over (detail::recursion_depth_limit(), 128 by default) for (std::size_t depth = 0; depth <= 300; ++depth) { CAPTURE(depth) @@ -396,7 +396,7 @@ TEST_CASE("merge_patch() with an argument that aliases *this (#5641)") SECTION("a patch nested past the iterative descent bound aliases *this") { // every depth on either side of where the iterative version takes - // over (detail::recursion_depth_limit(), 128); patching *this with + // over (detail::recursion_depth_limit(), 128 by default); patching *this with // itself is idempotent, aliased or not for (const std::size_t depth : { diff --git a/tests/src/unit-modifiers.cpp b/tests/src/unit-modifiers.cpp index 3377eace0..0963b9881 100644 --- a/tests/src/unit-modifiers.cpp +++ b/tests/src/unit-modifiers.cpp @@ -1124,7 +1124,7 @@ TEST_CASE("update() on deeply nested values") SECTION("merging past the descent bound gives the same result") { // every depth on either side of where the iterative version takes - // over (detail::recursion_depth_limit(), 128) + // over (detail::recursion_depth_limit(), 128 by default) for (std::size_t depth = 0; depth <= 300; ++depth) { CAPTURE(depth) @@ -1215,7 +1215,7 @@ TEST_CASE("update() with an argument that aliases *this (#5641)") SECTION("nested past the iterative descent bound aliases *this") { // every depth on either side of where the iterative version takes - // over (detail::recursion_depth_limit(), 128); merging *this into + // over (detail::recursion_depth_limit(), 128 by default); merging *this into // itself is idempotent, aliased or not for (const std::size_t depth : { diff --git a/tests/src/unit-recursion_depth_limit.cpp b/tests/src/unit-recursion_depth_limit.cpp new file mode 100644 index 000000000..cf62d1218 --- /dev/null +++ b/tests/src/unit-recursion_depth_limit.cpp @@ -0,0 +1,614 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#include "doctest_compatibility.h" + +#include +using nlohmann::json; + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// JSON_RECURSION_DEPTH_LIMIT changes only how much call stack the walks over a +// value use, never what they return. This file checks both: the macro is +// honoured (the number of levels the containers handle themselves is the limit) +// and every walk returns what it returns with the default limit. The expected +// results are built here, without the library, so that the same file passes for +// every limit; tests/CMakeLists.txt builds it with limits 0, 1, and 254 besides +// the default. Values are nested up to 300 levels (and past every limit, so that +// the explicit stack of every walk is exercised, even for the limit of 254) and +// are built and taken apart without recursion. Keep the depths moderate: walks +// like diff and dump with indentation are quadratic in the depth. + +namespace +{ +constexpr std::size_t limit() +{ + return nlohmann::detail::recursion_depth_limit(); +} + +// a value nested as @a kinds says, outermost level first: 'a' is an array of +// one element, 'o' an object with the single key "a"; @a leaf is innermost +json build(const std::string& kinds, json leaf) +{ + json j = std::move(leaf); + for (std::size_t i = kinds.size(); i-- > 0;) + { + if (kinds[i] == 'a') + { + json a = json::array(); + a.push_back(std::move(j)); + j = std::move(a); + } + else + { + json o = json::object(); + o["a"] = std::move(j); + j = std::move(o); + } + } + return j; +} + +std::string repeat(const std::string& s, std::size_t n) +{ + std::string result; + for (std::size_t i = 0; i < n; ++i) + { + result += s; + } + return result; +} + +// the nesting of the given shape and depth: arrays, objects, or both alternating +std::string make_kinds(char shape, std::size_t depth) +{ + std::string kinds; + for (std::size_t i = 0; i < depth; ++i) + { + if (shape == 'm') + { + kinds += i % 2 == 0 ? 'a' : 'o'; + } + else + { + kinds += shape; + } + } + return kinds; +} + +std::string expected_compact(const std::string& kinds, const std::string& leaf) +{ + std::string s; + for (const char kind : kinds) + { + s += kind == 'a' ? "[" : "{\"a\":"; + } + s += leaf; + for (std::size_t i = kinds.size(); i-- > 0;) + { + s += kinds[i] == 'a' ? "]" : "}"; + } + return s; +} + +// dump(1) +std::string expected_indented(const std::string& kinds, const std::string& leaf) +{ + std::string s; + for (std::size_t i = 0; i < kinds.size(); ++i) + { + s += kinds[i] == 'a' ? "[\n" : "{\n"; + s += std::string(i + 1, ' '); + if (kinds[i] == 'o') + { + s += "\"a\": "; + } + } + s += leaf; + for (std::size_t i = kinds.size(); i-- > 0;) + { + s += "\n" + std::string(i, ' ') + (kinds[i] == 'a' ? "]" : "}"); + } + return s; +} + +// the JSON pointer to the innermost value +std::string pointer(const std::string& kinds) +{ + std::string s; + for (const char kind : kinds) + { + s += kind == 'a' ? "/0" : "/a"; + } + return s; +} + +// the binary formats: the bytes that open and close a level, and the leaf 1 +std::vector expected_cbor(const std::string& kinds) +{ + std::vector bytes; + for (const char kind : kinds) + { + if (kind == 'a') + { + bytes.push_back(0x81); + } + else + { + bytes.push_back(0xA1); + bytes.push_back(0x61); + bytes.push_back('a'); + } + } + bytes.push_back(0x01); + return bytes; +} + +std::vector expected_msgpack(const std::string& kinds) +{ + std::vector bytes; + for (const char kind : kinds) + { + if (kind == 'a') + { + bytes.push_back(0x91); + } + else + { + bytes.push_back(0x81); + bytes.push_back(0xA1); + bytes.push_back('a'); + } + } + bytes.push_back(0x01); + return bytes; +} + +std::vector expected_ubjson(const std::string& kinds) +{ + std::vector bytes; + for (const char kind : kinds) + { + if (kind == 'a') + { + bytes.push_back('['); + } + else + { + bytes.push_back('{'); + bytes.push_back('i'); + bytes.push_back(1); + bytes.push_back('a'); + } + } + bytes.push_back('i'); + bytes.push_back(1); + for (std::size_t i = kinds.size(); i-- > 0;) + { + bytes.push_back(kinds[i] == 'a' ? ']' : '}'); + } + return bytes; +} + +// the depths worth trying for the limit: either side of it, twice it (if that +// is not too deep to be quick), and a value nested well past all limits +std::vector depths() +{ + std::vector result; + result.push_back(1); + result.push_back(2); + result.push_back(limit() > 0 ? limit() - 1 : 3); + result.push_back(limit()); + result.push_back(limit() + 1); + if ((2 * limit()) + 5 <= 300) + { + result.push_back((2 * limit()) + 5); + } + result.push_back(300); + std::sort(result.begin(), result.end()); + result.erase(std::unique(result.begin(), result.end()), result.end()); + result.erase(std::remove(result.begin(), result.end(), static_cast(0)), result.end()); + return result; +} + +// the call for every shape and depth +void for_each_value(const std::function& check) +{ + const std::string shapes = "aom"; + for (const char shape : shapes) + { + for (const std::size_t depth : depths()) + { + CAPTURE(shape) + CAPTURE(depth) + check(make_kinds(shape, depth)); + } + } +} + +// An array type that counts how many of its copies are being made at once. A +// copy of a value copies one level at a time through the array type until the +// limit has been reached, so the peak is the number of levels that were copied +// with the call stack. +std::size_t copies_going_on = 0; +std::size_t copies_peak = 0; +std::size_t equal_going_on = 0; +std::size_t equal_peak = 0; + +template> +class counting_vector : public std::vector +{ + public: + counting_vector() = default; + + template + counting_vector(InputIt first, InputIt last) : std::vector(first, last) {} + + counting_vector(const counting_vector& other) : std::vector() + { + ++copies_going_on; + copies_peak = (std::max)(copies_peak, copies_going_on); + this->std::vector::operator=(static_cast&>(other)); + --copies_going_on; + } + + counting_vector(counting_vector&&) = default; + counting_vector& operator=(const counting_vector&) = default; + counting_vector& operator=(counting_vector&&) = default; + ~counting_vector() = default; + + friend bool operator==(const counting_vector& lhs, const counting_vector& rhs) + { + ++equal_going_on; + equal_peak = (std::max)(equal_peak, equal_going_on); + const bool result = static_cast&>(lhs) == static_cast&>(rhs); + --equal_going_on; + return result; + } +}; + +using counting_json = json::with_array_t; + +// A base class whose copy constructor copies a whole value of the same +// specialization when it copies a marked element, as user code may do from a +// hook. That copy runs while the copy that it is part of is going on. +struct copy_hook // NOLINT(cppcoreguidelines-special-member-functions,hicpp-special-member-functions) +{ + int mark = 0; + + copy_hook() = default; + copy_hook(const copy_hook& other); + copy_hook(copy_hook&&) = default; + copy_hook& operator=(const copy_hook&) = default; + copy_hook& operator=(copy_hook&&) = default; +}; + +using hook_json = nlohmann::basic_json, copy_hook>; + +const hook_json* hook_copy_source = nullptr; +int hook_copies_going_on = 0; + +copy_hook::copy_hook(const copy_hook& other) : mark(other.mark) +{ + // the value that is copied is marked, too: its copy copies it again, once + if (other.mark == 1 && hook_copy_source != nullptr && hook_copies_going_on < 2) + { + ++hook_copies_going_on; + const hook_json nested = *hook_copy_source; // NOLINT(performance-unnecessary-copy-initialization) + static_cast(nested); + --hook_copies_going_on; + } +} + +// an array nested @a depth levels around the integer 1, which is marked if @a marked +hook_json hook_nested(std::size_t depth, bool marked) +{ + hook_json j = 1; + if (marked) + { + static_cast(j).mark = 1; + } + for (std::size_t i = 0; i < depth; ++i) + { + hook_json a = hook_json::array(); + a.push_back(std::move(j)); + j = std::move(a); + } + return j; +} +} // namespace + +// the macro is the limit, and is in the range the library supports +static_assert(nlohmann::detail::recursion_depth_limit() == static_cast(JSON_RECURSION_DEPTH_LIMIT), + "the library does not use JSON_RECURSION_DEPTH_LIMIT"); +static_assert(nlohmann::detail::recursion_depth_limit() <= 254, "the nesting depth must fit in a byte"); + +// the macro must not be a character or bool: every character type is refused +static_assert(nlohmann::detail::recursion_depth_limit_is_char_or_bool::value, "bool is refused"); +static_assert(nlohmann::detail::recursion_depth_limit_is_char_or_bool::value, "char is refused"); +static_assert(nlohmann::detail::recursion_depth_limit_is_char_or_bool::value, "signed char is refused"); +static_assert(nlohmann::detail::recursion_depth_limit_is_char_or_bool::value, "unsigned char is refused"); +static_assert(nlohmann::detail::recursion_depth_limit_is_char_or_bool::value, "wchar_t is refused"); +static_assert(nlohmann::detail::recursion_depth_limit_is_char_or_bool::value, "char16_t is refused"); +static_assert(nlohmann::detail::recursion_depth_limit_is_char_or_bool::value, "char32_t is refused"); +#if defined(__cpp_char8_t) + static_assert(nlohmann::detail::recursion_depth_limit_is_char_or_bool::value, "char8_t is refused"); +#endif +static_assert(!nlohmann::detail::recursion_depth_limit_is_char_or_bool::value, "int is accepted"); +static_assert(!nlohmann::detail::recursion_depth_limit_is_char_or_bool::value, "unsigned long long is accepted"); + +TEST_CASE("JSON_RECURSION_DEPTH_LIMIT is honoured") +{ + SECTION("copying and comparing stop descending at the limit") + { + // copy and compare a value nested deeper than the limit, with the array type that counts + counting_json inner = 1; + for (std::size_t i = 0; i < (2 * limit()) + 5; ++i) + { + counting_json a = counting_json::array(); + a.push_back(std::move(inner)); + inner = std::move(a); + } + + copies_going_on = copies_peak = 0; + const counting_json copy = inner; + CHECK(copies_going_on == 0); + + equal_going_on = equal_peak = 0; + CHECK((inner == copy)); + CHECK(equal_going_on == 0); + +#ifdef JSON_NO_THREAD_LOCAL + // without the counters of the levels, no level is descended into + CHECK(copies_peak == 0); + CHECK(equal_peak == 0); +#else + // copying descends into as many levels as the limit says, and nothing further + CHECK(copies_peak == limit()); + CHECK(equal_peak == limit()); +#endif + + // a value that is not nested deeper than the limit is not cut short + const std::size_t shallow = limit() > 0 ? limit() : 1; + counting_json small = 1; + for (std::size_t i = 0; i < shallow; ++i) + { + counting_json a = counting_json::array(); + a.push_back(std::move(small)); + small = std::move(a); + } + copies_going_on = copies_peak = 0; + const counting_json small_copy = small; + CHECK(copies_going_on == 0); +#ifndef JSON_NO_THREAD_LOCAL + CHECK(copies_peak == limit()); +#endif + CHECK(small_copy.dump() == expected_compact(make_kinds('a', shallow), "1")); + } +} + +TEST_CASE("a copy made while a copy is going on is bounded, too") +{ + // The innermost element of the value is marked, and copying it (which happens + // after limit() levels have been entered, when the copy continues without the + // call stack) copies the value again, and that copy once more. Those copies + // must not be given limit() levels of their own: the count of the levels entered + // never exceeds the limit, not even by one, which for the largest limit would + // wrap the byte that holds it around to 0. + const hook_json marked = hook_nested((2 * limit()) + 5, true); + hook_copy_source = &marked; + + copies_going_on = copies_peak = 0; + const hook_json copy = marked; // NOLINT(performance-unnecessary-copy-initialization) + hook_copy_source = nullptr; + CHECK(copies_going_on == 0); + CHECK(static_cast(copy).mark == 0); // (the top level is not marked) + +#ifdef JSON_NO_THREAD_LOCAL + CHECK(copies_peak == 0); +#else + CHECK(copies_peak == limit()); +#endif +} + +TEST_CASE("walks over values nested around the recursion depth limit") +{ + SECTION("copying and comparing") + { + for_each_value([](const std::string & kinds) + { + const json a = build(kinds, 1); + const json b = a; // NOLINT(performance-unnecessary-copy-initialization) + CHECK((a == b)); + CHECK_FALSE((a != b)); + CHECK_FALSE((a < b)); + CHECK((a <= b)); + CHECK((a >= b)); + CHECK_FALSE((a > b)); + + json assigned; + assigned = a; + CHECK((assigned == a)); + + // differing only in the innermost value + const json c = build(kinds, 2); + CHECK((a != c)); + CHECK((a < c)); + CHECK((c > a)); + CHECK_FALSE((c < a)); + CHECK_FALSE((a == c)); + + // a same-shape array or object that is one level shallower + const json shallower = build(kinds.substr(1), 1); + CHECK((shallower != a)); + const json outer = build(kinds.substr(0, 1), 1); + CHECK((outer != a) == (kinds.size() > 1)); + + const json moved = std::move(assigned); + CHECK((moved == a)); + }); + } + + SECTION("serializing") + { + for_each_value([](const std::string & kinds) + { + const json a = build(kinds, 1); + const std::string compact = expected_compact(kinds, "1"); + const std::string indented = expected_indented(kinds, "1"); + + CHECK(a.dump() == compact); + CHECK(a.dump(1) == indented); + + std::ostringstream compact_stream; + compact_stream << a; + CHECK(compact_stream.str() == compact); + std::ostringstream indented_stream; + indented_stream << std::setw(1) << a; + CHECK(indented_stream.str() == indented); + + CHECK((json::parse(compact) == a)); + CHECK((json::parse(indented) == a)); + }); + } + + SECTION("hashing") + { + for_each_value([](const std::string & kinds) + { + const json a = build(kinds, 1); + const std::hash hash; + CHECK(hash(a) == hash(json(a))); + CHECK(hash(a) == hash(json::parse(expected_compact(kinds, "1")))); + CHECK(hash(a) != hash(build(kinds, 2))); + CHECK(hash(a) != hash(build(kinds.substr(1), 1))); + }); + } + + SECTION("diff and patch") + { + for_each_value([](const std::string & kinds) + { + const json a = build(kinds, 1); + const json b = build(kinds, 2); + + const json difference = json::diff(a, b); + CHECK(difference.dump() == "[{\"op\":\"replace\",\"path\":\"" + pointer(kinds) + "\",\"value\":2}]"); + CHECK((a.patch(difference) == b)); + CHECK((json::diff(a, a) == json::array())); + }); + } + + SECTION("flatten and unflatten") + { + for_each_value([](const std::string & kinds) + { + const json a = build(kinds, 1); + const json flat = a.flatten(); + CHECK(flat.dump() == "{\"" + pointer(kinds) + "\":1}"); + CHECK((flat.unflatten() == a)); + }); + } + + SECTION("merge_patch and update") + { + for (const std::size_t depth : depths()) + { + CAPTURE(depth) + const std::string kinds = make_kinds('o', depth); + const std::string all_but_leaf = repeat("{\"a\":", depth - 1); + + // the patch removes the innermost member, or replaces it + json target = build(kinds, 1); + target.merge_patch(build(kinds, json(nullptr))); + CHECK(target.dump() == all_but_leaf + "{}" + repeat("}", depth - 1)); + + target = build(kinds, 1); + target.merge_patch(build(kinds, 2)); + CHECK(target.dump() == expected_compact(kinds, "2")); + + // merging the objects of two values nested alike, and replacing the first level + const json base = build(kinds.substr(1), json::parse("{\"x\":1}")); + const json other = build(kinds.substr(1), json::parse("{\"y\":2}")); + json merged = json{{"a", base}}; + merged.update(json{{"a", other}}, true); + CHECK(merged.dump() == expected_compact(kinds, "{\"x\":1,\"y\":2}")); + + json replaced = json{{"a", base}}; + replaced.update(json{{"a", other}}); + CHECK(replaced.dump() == expected_compact(kinds, "{\"y\":2}")); + + // update from a range + json from_range = json{{"a", base}}; + const json source = json{{"a", other}}; + from_range.update(source.begin(), source.end(), true); + CHECK((from_range == merged)); + } + } + + SECTION("binary formats") + { + for_each_value([](const std::string & kinds) + { + const json a = build(kinds, 1); + + const std::vector cbor = json::to_cbor(a); + CHECK(cbor == expected_cbor(kinds)); + CHECK((json::from_cbor(cbor) == a)); + + const std::vector msgpack = json::to_msgpack(a); + CHECK(msgpack == expected_msgpack(kinds)); + CHECK((json::from_msgpack(msgpack) == a)); + + const std::vector ubjson = json::to_ubjson(a); + CHECK(ubjson == expected_ubjson(kinds)); + CHECK((json::from_ubjson(ubjson) == a)); + + CHECK((json::from_bjdata(json::to_bjdata(a)) == a)); + CHECK((json::from_bon8(json::to_bon8(a)) == a)); + + // BSON documents are objects + const json document = json{{"a", a}}; + CHECK((json::from_bson(json::to_bson(document)) == document)); + }); + } + + SECTION("conversion to another specialization") + { + for_each_value([](const std::string & kinds) + { + const json a = build(kinds, 1); + const nlohmann::ordered_json ordered = a; + CHECK(ordered.dump() == expected_compact(kinds, "1")); + const json back = ordered; + CHECK((back == a)); + }); + } + + SECTION("destroying") + { + for_each_value([](const std::string & kinds) + { + json a = build(kinds, 1); + a = nullptr; + CHECK(a.is_null()); + }); + } +} diff --git a/tests/src/unit-regression1.cpp b/tests/src/unit-regression1.cpp index 367f52dfa..fcb533c5d 100644 --- a/tests/src/unit-regression1.cpp +++ b/tests/src/unit-regression1.cpp @@ -1554,8 +1554,10 @@ TEST_CASE("regression tests 1") SECTION("a missing member of an object") { my_json j = make(); -#ifdef JSON_NO_THREAD_LOCAL - // (set by the user, or by the library for clang with MinGW) +#if defined(JSON_NO_THREAD_LOCAL) || JSON_RECURSION_DEPTH_LIMIT < 2 + // (set by the user, or by the library for clang with MinGW; or fewer than two + // levels are copied by the object type itself, as for a recursion depth limit + // of 0 or 1: the objects below the outermost one are the second level) // the objects below the outermost one are not copied by their // object_t here, but member by member, which keeps their // timestamps correct: "c" is not mistaken for "b" and is inserted