From d1377a1e843857b5de04ea8258c0725af8469c28 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Thu, 15 Oct 2026 09:40:53 +0200 Subject: [PATCH] Add benchmarks for SAX parsing, value operations, JSON Pointer, and conversions The benchmark suite covered only parsing, serialization, and the binary formats. Add benchmarks for SAX parsing without building values, copying, destroying, comparing and iterating documents, flatten()/unflatten() (also for ordered_json), diff() and patch(), JSON Pointer parsing, printing and lookup, and conversions from and to standard containers. They use only API that 3.12.0 also has, so releases can be compared on them. Signed-off-by: Niels Lohmann --- tests/benchmarks/README.md | 7 + tests/benchmarks/src/benchmarks.cpp | 523 ++++++++++++++++++++++++++++ 2 files changed, 530 insertions(+) diff --git a/tests/benchmarks/README.md b/tests/benchmarks/README.md index 6f067db60..283b156e5 100644 --- a/tests/benchmarks/README.md +++ b/tests/benchmarks/README.md @@ -16,6 +16,13 @@ Micro-benchmarks for parsing, serialization and the binary formats, written with | `FromBinaryBuffer`, `FromBinaryFile` | read CBOR, MessagePack, UBJSON, BJData and BSON from a buffer or a `FILE*` | | `FromBinaryShape` | read deeply nested, container-heavy and scalar-heavy documents in every binary format | | `FromCborChunkedString` | read CBOR strings split into indefinite-length chunks | +| `SaxParse` | parse with a SAX handler that only counts events, so no values are built | +| `Copy`, `Destroy`, `Equal`, `Iterate` | copy, destroy, or compare a parsed document, or visit every value with range-based `for` loops | +| `Flatten`, `FlattenOrdered`, `Unflatten` | `flatten()` for `json` and `ordered_json`, and `unflatten()` | +| `Diff`, `Patch` | `diff()` against the document with every eighth scalar changed, and applying that patch | +| `JsonPointerParse`, `JsonPointerToString`, `JsonPointerAt`, `JsonPointerContains` | the JSON pointers of every scalar in a document: parse them from strings, turn them back into strings, look them up with `at()` and `contains()` | +| `GetVectorInt`, `GetVectorDouble`, `GetMap` | `get>()`, `get>()` and `get>()` on large arrays and a large flat object | +| `ToJsonStrings` | create a `json` array from a `std::vector` of 100,000 strings, copied or moved | The input files are those of [nativejson-benchmark](https://github.com/miloyip/nativejson-benchmark) (`canada`, `citm_catalog`, `twitter`), a large `jeopardy` file, and number-heavy files (`floats`, `signed_ints`, ...). diff --git a/tests/benchmarks/src/benchmarks.cpp b/tests/benchmarks/src/benchmarks.cpp index 5df7dd473..01b4c92a3 100644 --- a/tests/benchmarks/src/benchmarks.cpp +++ b/tests/benchmarks/src/benchmarks.cpp @@ -8,8 +8,11 @@ #include #include +#include #include +#include #include +#include #include #include @@ -621,4 +624,524 @@ static void FromCborChunkedString(benchmark::State& state, const std::size_t chu BENCHMARK_CAPTURE(FromCborChunkedString, 10000 chunks, 10000); +////////////////////////////////////////////////////////////////////////////// +// SAX-parse JSON from string +// +// sax_parse() with a handler that does next to nothing runs the lexer and +// parser without building values, so the gap to ParseString above is the cost +// of building the DOM. +////////////////////////////////////////////////////////////////////////////// + +static std::string read_file(const char* filename) +{ + std::ifstream f(filename); + return std::string((std::istreambuf_iterator(f)), std::istreambuf_iterator()); +} + +static json load_json(const char* filename) +{ + return json::parse(read_file(filename)); +} + +/// SAX handler that only counts the events, so that the handler costs next to nothing +class counting_sax : public nlohmann::json_sax +{ + public: + std::size_t events = 0; + + bool null() override + { + return count(); + } + bool boolean(bool /*val*/) override + { + return count(); + } + bool number_integer(number_integer_t /*val*/) override + { + return count(); + } + bool number_unsigned(number_unsigned_t /*val*/) override + { + return count(); + } + bool number_float(number_float_t /*val*/, const string_t& /*s*/) override + { + return count(); + } + bool string(string_t& /*val*/) override + { + return count(); + } + bool binary(binary_t& /*val*/) override + { + return count(); + } + bool start_object(std::size_t /*elements*/) override + { + return count(); + } + bool key(string_t& /*val*/) override + { + return count(); + } + bool end_object() override + { + return count(); + } + bool start_array(std::size_t /*elements*/) override + { + return count(); + } + bool end_array() override + { + return count(); + } + bool parse_error(std::size_t /*position*/, const std::string& /*last_token*/, const json::exception& /*ex*/) override + { + return false; + } + + private: + bool count() + { + ++events; + return true; + } +}; + +static void SaxParse(benchmark::State& state, const char* filename) +{ + const std::string str = read_file(filename); + + for (auto _ : state) + { + counting_sax sax; + bool result = json::sax_parse(str, &sax); + benchmark::DoNotOptimize(result); + benchmark::DoNotOptimize(sax.events); + } + + state.SetBytesProcessed(state.iterations() * str.size()); +} +BENCHMARK_CAPTURE(SaxParse, canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json"); +BENCHMARK_CAPTURE(SaxParse, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json"); +BENCHMARK_CAPTURE(SaxParse, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json"); + +////////////////////////////////////////////////////////////////////////////// +// work with values: copy, destroy, compare, iterate +// +// The parsed test files stand in for a document an application keeps around. +// Each benchmark times one whole-document operation; setup and cleanup run +// outside the timed section. +////////////////////////////////////////////////////////////////////////////// + +static void Copy(benchmark::State& state, const char* filename) +{ + const json j = load_json(filename); + + for (auto _ : state) + { + auto* copy = new json(j); + + state.PauseTiming(); + delete copy; + state.ResumeTiming(); + } +} +BENCHMARK_CAPTURE(Copy, canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json"); +BENCHMARK_CAPTURE(Copy, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json"); +BENCHMARK_CAPTURE(Copy, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json"); + +static void Destroy(benchmark::State& state, const char* filename) +{ + const json j = load_json(filename); + + for (auto _ : state) + { + state.PauseTiming(); + auto* copy = new json(j); + state.ResumeTiming(); + + delete copy; + } +} +BENCHMARK_CAPTURE(Destroy, canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json"); +BENCHMARK_CAPTURE(Destroy, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json"); +BENCHMARK_CAPTURE(Destroy, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json"); + +static void Equal(benchmark::State& state, const char* filename) +{ + // two separate values, so that the comparison cannot stop at identical addresses + const json j = load_json(filename); + const json copy = j; + + for (auto _ : state) + { + bool result = (j == copy); + benchmark::DoNotOptimize(result); + } +} +BENCHMARK_CAPTURE(Equal, canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json"); +BENCHMARK_CAPTURE(Equal, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json"); +BENCHMARK_CAPTURE(Equal, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json"); + +/// visit every value with range-based for loops, as client code walks a document +static std::size_t count_values(const json& j) +{ + std::size_t result = 1; + if (j.is_structured()) + { + for (const auto& element : j) + { + result += count_values(element); + } + } + return result; +} + +static void Iterate(benchmark::State& state, const char* filename) +{ + const json j = load_json(filename); + + for (auto _ : state) + { + std::size_t result = count_values(j); + benchmark::DoNotOptimize(result); + } + + state.SetItemsProcessed(state.iterations() * static_cast(count_values(j))); +} +BENCHMARK_CAPTURE(Iterate, canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json"); +BENCHMARK_CAPTURE(Iterate, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json"); +BENCHMARK_CAPTURE(Iterate, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json"); + +////////////////////////////////////////////////////////////////////////////// +// flatten and unflatten +////////////////////////////////////////////////////////////////////////////// + +static void Flatten(benchmark::State& state, const char* filename) +{ + const json j = load_json(filename); + + for (auto _ : state) + { + json flat = j.flatten(); + benchmark::DoNotOptimize(flat); + } +} +BENCHMARK_CAPTURE(Flatten, canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json"); +BENCHMARK_CAPTURE(Flatten, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json"); +BENCHMARK_CAPTURE(Flatten, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json"); + +// ordered_json objects look keys up linearly, which flatten() used to do for +// every value; canada (330,000 values in one flat object) is left out, as it +// would take minutes with such versions +static void FlattenOrdered(benchmark::State& state, const char* filename) +{ + const nlohmann::ordered_json j = nlohmann::ordered_json::parse(read_file(filename)); + + for (auto _ : state) + { + nlohmann::ordered_json flat = j.flatten(); + benchmark::DoNotOptimize(flat); + } +} +BENCHMARK_CAPTURE(FlattenOrdered, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json"); +BENCHMARK_CAPTURE(FlattenOrdered, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json"); + +static void Unflatten(benchmark::State& state, const char* filename) +{ + const json flat = load_json(filename).flatten(); + + for (auto _ : state) + { + json j = flat.unflatten(); + benchmark::DoNotOptimize(j); + } +} +BENCHMARK_CAPTURE(Unflatten, canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json"); +BENCHMARK_CAPTURE(Unflatten, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json"); +BENCHMARK_CAPTURE(Unflatten, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json"); + +////////////////////////////////////////////////////////////////////////////// +// diff and patch +// +// The target is the document with every eighth scalar changed, so the patch +// has one "replace" operation per changed value and the structure stays the +// same. +////////////////////////////////////////////////////////////////////////////// + +static void change_every_eighth_scalar(json& j, std::size_t& counter) +{ + if (j.is_structured()) + { + for (auto& element : j) + { + change_every_eighth_scalar(element, counter); + } + return; + } + + if (++counter % 8 != 0) + { + return; + } + + switch (j.type()) + { + case json::value_t::number_integer: + j = j.get() + 1; + break; + case json::value_t::number_unsigned: + j = j.get() + 1; + break; + case json::value_t::number_float: + j = j.get() + 0.5; + break; + case json::value_t::string: + j = j.get() + "!"; + break; + case json::value_t::boolean: + j = !j.get(); + break; + case json::value_t::null: + case json::value_t::object: + case json::value_t::array: + case json::value_t::binary: + case json::value_t::discarded: + default: + j = 0; + break; + } +} + +static json changed_copy(const json& j) +{ + json result = j; + std::size_t counter = 0; + change_every_eighth_scalar(result, counter); + return result; +} + +static void Diff(benchmark::State& state, const char* filename) +{ + const json source = load_json(filename); + const json target = changed_copy(source); + + for (auto _ : state) + { + json patch = json::diff(source, target); + benchmark::DoNotOptimize(patch); + } + + state.SetItemsProcessed(state.iterations() * static_cast(json::diff(source, target).size())); +} +BENCHMARK_CAPTURE(Diff, canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json"); +BENCHMARK_CAPTURE(Diff, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json"); +BENCHMARK_CAPTURE(Diff, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json"); + +static void Patch(benchmark::State& state, const char* filename) +{ + const json source = load_json(filename); + const json patch = json::diff(source, changed_copy(source)); + + for (auto _ : state) + { + json result = source.patch(patch); + benchmark::DoNotOptimize(result); + } + + state.SetItemsProcessed(state.iterations() * static_cast(patch.size())); +} +BENCHMARK_CAPTURE(Patch, canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json"); +BENCHMARK_CAPTURE(Patch, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json"); +BENCHMARK_CAPTURE(Patch, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json"); + +////////////////////////////////////////////////////////////////////////////// +// JSON Pointer +// +// The pointers are those of every scalar in the document, i.e., the keys of +// its flattened form. +////////////////////////////////////////////////////////////////////////////// + +static std::vector pointer_strings(const json& j) +{ + std::vector result; + const json flat = j.flatten(); + result.reserve(flat.size()); + for (auto it = flat.begin(); it != flat.end(); ++it) + { + result.push_back(it.key()); + } + return result; +} + +static std::vector pointers(const json& j) +{ + const std::vector strings = pointer_strings(j); + std::vector result; + result.reserve(strings.size()); + for (const auto& s : strings) + { + result.emplace_back(s); + } + return result; +} + +static void JsonPointerParse(benchmark::State& state, const char* filename) +{ + const std::vector strings = pointer_strings(load_json(filename)); + + for (auto _ : state) + { + for (const auto& s : strings) + { + json::json_pointer ptr(s); + benchmark::DoNotOptimize(ptr); + } + } + + state.SetItemsProcessed(state.iterations() * static_cast(strings.size())); +} +BENCHMARK_CAPTURE(JsonPointerParse, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json"); +BENCHMARK_CAPTURE(JsonPointerParse, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json"); + +static void JsonPointerToString(benchmark::State& state, const char* filename) +{ + const std::vector ptrs = pointers(load_json(filename)); + + for (auto _ : state) + { + for (const auto& ptr : ptrs) + { + std::string s = ptr.to_string(); + benchmark::DoNotOptimize(s); + } + } + + state.SetItemsProcessed(state.iterations() * static_cast(ptrs.size())); +} +BENCHMARK_CAPTURE(JsonPointerToString, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json"); +BENCHMARK_CAPTURE(JsonPointerToString, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json"); + +static void JsonPointerAt(benchmark::State& state, const char* filename) +{ + const json j = load_json(filename); + const std::vector ptrs = pointers(j); + + for (auto _ : state) + { + for (const auto& ptr : ptrs) + { + const json& value = j.at(ptr); + benchmark::DoNotOptimize(&value); + } + } + + state.SetItemsProcessed(state.iterations() * static_cast(ptrs.size())); +} +BENCHMARK_CAPTURE(JsonPointerAt, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json"); +BENCHMARK_CAPTURE(JsonPointerAt, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json"); + +static void JsonPointerContains(benchmark::State& state, const char* filename) +{ + const json j = load_json(filename); + const std::vector ptrs = pointers(j); + + for (auto _ : state) + { + for (const auto& ptr : ptrs) + { + bool result = j.contains(ptr); + benchmark::DoNotOptimize(result); + } + } + + state.SetItemsProcessed(state.iterations() * static_cast(ptrs.size())); +} +BENCHMARK_CAPTURE(JsonPointerContains, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json"); +BENCHMARK_CAPTURE(JsonPointerContains, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json"); + +////////////////////////////////////////////////////////////////////////////// +// conversions from and to C++ containers +////////////////////////////////////////////////////////////////////////////// + +static void GetVectorInt(benchmark::State& state, const char* filename) +{ + const json j = load_json(filename); + + for (auto _ : state) + { + auto v = j.get>(); + benchmark::DoNotOptimize(v); + } + + state.SetItemsProcessed(state.iterations() * static_cast(j.size())); +} +BENCHMARK_CAPTURE(GetVectorInt, signed_ints, TEST_DATA_DIRECTORY "/regression/signed_ints.json"); + +static void GetVectorDouble(benchmark::State& state, const char* filename) +{ + const json j = load_json(filename); + + for (auto _ : state) + { + auto v = j.get>(); + benchmark::DoNotOptimize(v); + } + + state.SetItemsProcessed(state.iterations() * static_cast(j.size())); +} +BENCHMARK_CAPTURE(GetVectorDouble, floats, TEST_DATA_DIRECTORY "/regression/floats.json"); + +// a large flat object: the flattened document, one member per scalar +static void GetMap(benchmark::State& state, const char* filename) +{ + const json j = load_json(filename).flatten(); + + for (auto _ : state) + { + auto m = j.get>(); + benchmark::DoNotOptimize(m); + } + + state.SetItemsProcessed(state.iterations() * static_cast(j.size())); +} +BENCHMARK_CAPTURE(GetMap, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json"); + +/// 100,000 strings, each too long for the small string optimization +static std::vector make_strings() +{ + std::vector result; + result.reserve(100000); + for (std::size_t i = 0; i < 100000; ++i) + { + result.push_back("a string that does not fit in place: " + std::to_string(i)); + } + return result; +} + +static void ToJsonStrings(benchmark::State& state, const bool move) +{ + const std::vector strings = make_strings(); + + for (auto _ : state) + { + state.PauseTiming(); + std::vector input = strings; + state.ResumeTiming(); + + auto* j = move ? new json(std::move(input)) : new json(input); + + state.PauseTiming(); + delete j; + state.ResumeTiming(); + } + + state.SetItemsProcessed(state.iterations() * static_cast(strings.size())); +} +BENCHMARK_CAPTURE(ToJsonStrings, copy, false); +BENCHMARK_CAPTURE(ToJsonStrings, move, true); + BENCHMARK_MAIN();