Pin Google Benchmark to release 1.9.5

The benchmarks fetched Google Benchmark's main branch, so two builds on
different days could measure with different library code, and CMake 3.30
and later warn that the single-argument FetchContent_Populate() is
deprecated. Fetch the 1.9.5 release archive, verified by its SHA-256,
with FetchContent_MakeAvailable() instead. That needs CMake 3.14; Google
Benchmark itself already needed 3.13.

Its -Werror is switched off, so a newer compiler's new warnings cannot
break the pinned release, and its install rules are no longer added.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

cmake/ci.cmake

@@ -615,7 +615,7 @@ add_custom_target(ci_single_binaries
 add_custom_target(ci_benchmarks
     COMMAND ${CMAKE_COMMAND}
         -DCMAKE_BUILD_TYPE=Release -GNinja
-        -S${PROJECT_SOURCE_DIR}/benchmarks -B${PROJECT_BINARY_DIR}/build_benchmarks
+        -S${PROJECT_SOURCE_DIR}/tests/benchmarks -B${PROJECT_BINARY_DIR}/build_benchmarks
     COMMAND ${CMAKE_COMMAND} --build ${PROJECT_BINARY_DIR}/build_benchmarks --target json_benchmarks
     COMMAND cd ${PROJECT_BINARY_DIR}/build_benchmarks && ./json_benchmarks
     COMMENT "Run benchmarks"

cmake/download_test_data.cmake

@@ -77,7 +77,7 @@ if(CMAKE_CROSSCOMPILING)
 endif()
 if(NOT DEFINED LIBCPP_VERSION_OUTPUT_CACHED)
     try_run(RUN_RESULT_VAR COMPILE_RESULT_VAR
-        "${CMAKE_BINARY_DIR}" SOURCES "${CMAKE_SOURCE_DIR}/cmake/detect_libcpp_version.cpp"
+        "${CMAKE_BINARY_DIR}" SOURCES "${CMAKE_CURRENT_LIST_DIR}/detect_libcpp_version.cpp"
         RUN_OUTPUT_VARIABLE LIBCPP_VERSION_OUTPUT
         COMPILE_OUTPUT_VARIABLE LIBCPP_VERSION_COMPILE_OUTPUT
     )

tests/benchmarks/CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.11...3.14)
+cmake_minimum_required(VERSION 3.14)
 project(JSON_Benchmarks LANGUAGES CXX)
 
 # set compiler flags
@@ -6,29 +6,35 @@ if((CMAKE_CXX_COMPILER_ID MATCHES GNU) OR (CMAKE_CXX_COMPILER_ID MATCHES Clang))
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -flto -DNDEBUG -O3")
 endif()
 
-# configure Google Benchmarks
+# configure Google Benchmark; a fixed release, so that results stay comparable
+set(JSON_GOOGLE_BENCHMARK_VERSION 1.9.5)
 include(FetchContent)
-FetchContent_Declare(
-    benchmark
-    GIT_REPOSITORY https://github.com/google/benchmark.git
-    GIT_TAG origin/main
-    GIT_SHALLOW TRUE
-)
 
-FetchContent_GetProperties(benchmark)
+# only the library is needed; -Werror would break the pinned release as soon as
-if(NOT benchmark_POPULATED)
+# a newer compiler adds a warning
-    FetchContent_Populate(benchmark)
+set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "" FORCE)
-    set(BENCHMARK_ENABLE_TESTING OFF CACHE INTERNAL "" FORCE)
+set(BENCHMARK_ENABLE_INSTALL OFF CACHE BOOL "" FORCE)
-    add_subdirectory(${benchmark_SOURCE_DIR} ${benchmark_BINARY_DIR})
+set(BENCHMARK_ENABLE_WERROR OFF CACHE BOOL "" FORCE)
-endif()
+
+FetchContent_Declare(benchmark
+    URL https://github.com/google/benchmark/archive/refs/tags/v${JSON_GOOGLE_BENCHMARK_VERSION}.tar.gz
+    URL_HASH SHA256=9631341c82bac4a288bef951f8b26b41f69021794184ece969f8473977eaa340
+    DOWNLOAD_EXTRACT_TIMESTAMP TRUE
+)
+FetchContent_MakeAvailable(benchmark)
 
 # download test data
 set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../../cmake ${CMAKE_MODULE_PATH})
 include(download_test_data)
 
+# the header to benchmark; point this at a directory holding another version's
+# nlohmann/json.hpp to compare versions (see README.md)
+set(JSON_BENCHMARK_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../single_include" CACHE PATH
+    "directory containing the nlohmann/json.hpp to benchmark")
+
 # benchmark binary
 add_executable(json_benchmarks src/benchmarks.cpp)
 target_compile_features(json_benchmarks PRIVATE cxx_std_11)
 target_link_libraries(json_benchmarks benchmark ${CMAKE_THREAD_LIBS_INIT})
 add_dependencies(json_benchmarks download_test_data)
-target_include_directories(json_benchmarks PRIVATE ${CMAKE_SOURCE_DIR}/../../single_include ${CMAKE_BINARY_DIR}/include)
+target_include_directories(json_benchmarks PRIVATE ${JSON_BENCHMARK_INCLUDE_DIR} ${CMAKE_BINARY_DIR}/include)

tests/benchmarks/README.md

@@ -0,0 +1,130 @@
+# Benchmarks
+
+Micro-benchmarks for parsing, serialization and the binary formats, written with
+[Google Benchmark](https://github.com/google/benchmark). They are not run by CI; see
+[When to run them](#when-to-run-them).
+
+## What is measured
+
+| benchmark | what it does |
+|---|---|
+| `ParseFile`, `ParseString` | parse JSON from a file stream or a string |
+| `ParseIndented` | parse the large files re-indented by 4 spaces, for the lexer's whitespace handling |
+| `Dump` | serialize, compact (`-`) and indented (`4`) |
+| `ToCbor`, `BinaryToCbor` | write CBOR; `BinaryToCbor` writes binary values of growing size |
+| `FromMsgpack` | read MessagePack; unchanged over the years, so its numbers stay comparable across releases |
+| `FromBinaryBuffer`, `FromBinaryFile` | read CBOR, MessagePack, UBJSON, BJData and BSON from a buffer or a `FILE*` |
+| `FromBinaryShape` | read deeply nested, container-heavy and scalar-heavy documents in every binary format |
+| `FromCborChunkedString` | read CBOR strings split into indefinite-length chunks |
+
+The input files are those of [nativejson-benchmark](https://github.com/miloyip/nativejson-benchmark) (`canada`,
+`citm_catalog`, `twitter`), a large `jeopardy` file, and number-heavy files (`floats`, `signed_ints`, ...).
+`bytes_per_second` counts the bytes read or written: the JSON text when parsing, the output when serializing.
+
+## Requirements
+
+- CMake 3.14 or later, a C++11 compiler, and Ninja for the `make` target.
+- Network access on the first configure: CMake downloads Google Benchmark and the
+  [test data](https://github.com/nlohmann/json_test_data) into the build directory. To reuse a download of the test
+  data, pass `-DJSON_TestDataDirectory=<build directory>/test_files`.
+- Google Benchmark is pinned to a release (1.9.5), so that results from different days stay comparable. To update it,
+  change `JSON_GOOGLE_BENCHMARK_VERSION` and the archive's `URL_HASH` in `CMakeLists.txt` together.
+- The benchmarks include `single_include/nlohmann/json.hpp`, so run `make amalgamate` after changing anything in
+  `include/`.
+
+GCC and Clang builds use `-O3 -flto -DNDEBUG`.
+
+## Running them
+
+From the repository root, this builds everything from scratch in `cmake-build-benchmarks` and runs all benchmarks:
+
+```sh
+make run_benchmarks
+```
+
+To build once and run selectively:
+
+```sh
+cmake -S tests/benchmarks -B build-benchmarks -G Ninja -DCMAKE_BUILD_TYPE=Release
+cmake --build build-benchmarks
+build-benchmarks/json_benchmarks --benchmark_filter='ParseString|Dump'
+```
+
+Useful options of `json_benchmarks`:
+
+| option | effect |
+|---|---|
+| `--benchmark_list_tests` | list the benchmarks instead of running them |
+| `--benchmark_filter=<regex>` | run only the benchmarks whose names match |
+| `--benchmark_repetitions=<n>` | run every benchmark `n` times and add mean, median, standard deviation and coefficient of variation |
+| `--benchmark_enable_random_interleaving=true` | run the repetitions in random order, which spreads out drifts such as thermal throttling |
+| `--benchmark_min_time=<seconds>s` | run each benchmark at least this long (e.g. `2s`) |
+| `--benchmark_out=<file> --benchmark_out_format=json` | also write the results to a file, e.g. for `compare.py` |
+
+## Reading the output
+
+Each line shows the wall-clock `Time` and the `CPU` time per iteration, the number of `Iterations` Google Benchmark
+chose, and the throughput in `bytes_per_second`. With repetitions, the lines ending in `_median` are the ones to
+compare. A `_cv` (coefficient of variation) above a few percent means the machine was too noisy for small
+differences to mean anything.
+
+## Comparing two versions
+
+To see what a change or a release did, build the same benchmarks twice: once against the header of the version to
+compare with, and once against the current one. `JSON_BENCHMARK_INCLUDE_DIR` names the directory holding the
+`nlohmann/json.hpp` to benchmark. For example, to compare the current checkout with 3.12.0:
+
+```sh
+# the header of the version to compare with
+mkdir -p build-baseline-header/nlohmann
+git show v3.12.0:single_include/nlohmann/json.hpp > build-baseline-header/nlohmann/json.hpp
+
+# the same benchmarks, built against either header
+cmake -S tests/benchmarks -B build-baseline -G Ninja -DCMAKE_BUILD_TYPE=Release \
+      -DJSON_BENCHMARK_INCLUDE_DIR="$PWD/build-baseline-header"
+cmake -S tests/benchmarks -B build-current -G Ninja -DCMAKE_BUILD_TYPE=Release
+cmake --build build-baseline
+cmake --build build-current
+
+# run both, back to back
+build-baseline/json_benchmarks --benchmark_repetitions=10 --benchmark_enable_random_interleaving=true \
+                               --benchmark_out=build-baseline/results.json --benchmark_out_format=json
+build-current/json_benchmarks --benchmark_repetitions=10 --benchmark_enable_random_interleaving=true \
+                              --benchmark_out=build-current/results.json --benchmark_out_format=json
+```
+
+Google Benchmark ships a tool to compare the two result files. It needs NumPy and SciPy:
+
+```sh
+python3 -m venv build-venv
+build-venv/bin/pip install numpy scipy
+build-venv/bin/python build-current/_deps/benchmark-src/tools/compare.py -a benchmarks build-baseline/results.json build-current/results.json
+```
+
+The tool's own `tools/requirements.txt` pins NumPy and SciPy versions that need Python 3.11 or later; with an older
+Python, unpinned versions work as well. In its output:
+
+- the `Time` and `CPU` columns are relative changes: `-0.35` means 35% faster, `+0.10` means 10% slower;
+- `_pvalue` lines report a Mann-Whitney U test of whether the two versions differ. It needs at least 9
+  repetitions, and a p-value below 0.05 means the difference is unlikely to be noise;
+- `OVERALL_GEOMEAN` summarizes all benchmarks;
+- `-a` shows only the aggregates, not every repetition.
+
+The header you compare with must support everything the benchmarks use. The current benchmarks build against 3.12.0.
+Only benchmarks present in both result files are compared, so for older releases, either filter the benchmarks or
+build that release's own `tests/benchmarks` against its own header.
+
+## Getting stable numbers
+
+- Build and run both versions on the same machine, one right after the other.
+- Keep the machine otherwise idle: no builds, no browser, and a laptop plugged in.
+- On Linux, set the CPU frequency governor to `performance`, e.g. `sudo cpupower frequency-set --governor performance`.
+  Google Benchmark prints a warning when frequency scaling is enabled. Pinning the process to a core
+  (`taskset -c 2 ...`) helps as well.
+- Use 10 or more repetitions with random interleaving, compare medians, and treat changes within the `_cv` as noise.
+
+## When to run them
+
+They are a manual step, not part of CI: shared CI runners vary more between runs than most of the effects measured.
+Run the comparison above before a release, comparing the previous release tag with `develop`, and for pull requests
+that claim to change performance.

tests/benchmarks/src/benchmarks.cpp

@@ -131,7 +131,8 @@ static void Dump(benchmark::State& state, const char* filename, int indent)
 
     while (state.KeepRunning())
     {
-        j.dump(indent);
+        std::string output = j.dump(indent);
+        benchmark::DoNotOptimize(output);
     }
 
     state.SetBytesProcessed(state.iterations() * j.dump(indent).size());