mirror of
https://github.com/nlohmann/json.git
synced 2026-10-15 18:57:13 +00:00
Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Niels Lohmann
|
85d83bed74 | ||
|
Niels Lohmann
|
0905602724 | ||
|
Niels Lohmann
|
15b0ff3be6 | ||
|
Niels Lohmann
|
e4cb826600 | ||
|
Niels Lohmann
|
9dc60905ed | ||
|
Niels Lohmann
|
467670aa7a |
No files matched your search
@@ -16,13 +16,6 @@ Micro-benchmarks for parsing, serialization and the binary formats, written with
|
||||
| `FromBinaryBuffer`, `FromBinaryFile` | read CBOR, MessagePack, UBJSON, BJData and BSON from a buffer or a `FILE*` |
|
||||
| `FromBinaryShape` | read deeply nested, container-heavy and scalar-heavy documents in every binary format |
|
||||
| `FromCborChunkedString` | read CBOR strings split into indefinite-length chunks |
|
||||
| `SaxParse` | parse with a SAX handler that only counts events, so no values are built |
|
||||
| `Copy`, `Destroy`, `Equal`, `Iterate` | copy, destroy, or compare a parsed document, or visit every value with range-based `for` loops |
|
||||
| `Flatten`, `FlattenOrdered`, `Unflatten` | `flatten()` for `json` and `ordered_json`, and `unflatten()` |
|
||||
| `Diff`, `Patch` | `diff()` against the document with every eighth scalar changed, and applying that patch |
|
||||
| `JsonPointerParse`, `JsonPointerToString`, `JsonPointerAt`, `JsonPointerContains` | the JSON pointers of every scalar in a document: parse them from strings, turn them back into strings, look them up with `at()` and `contains()` |
|
||||
| `GetVectorInt`, `GetVectorDouble`, `GetMap` | `get<std::vector<std::int64_t>>()`, `get<std::vector<double>>()` and `get<std::map<std::string, json>>()` on large arrays and a large flat object |
|
||||
| `ToJsonStrings` | create a `json` array from a `std::vector` of 100,000 strings, copied or moved |
|
||||
|
||||
The input files are those of [nativejson-benchmark](https://github.com/miloyip/nativejson-benchmark) (`canada`,
|
||||
`citm_catalog`, `twitter`), a large `jeopardy` file, and number-heavy files (`floats`, `signed_ints`, ...).
|
||||
|
||||
@@ -8,14 +8,11 @@
|
||||
|
||||
#include <benchmark/benchmark.h>
|
||||
#include <nlohmann/json.hpp>
|
||||
#include <array>
|
||||
#include <cstdint>
|
||||
#include <deque>
|
||||
#include <fstream>
|
||||
#include <map>
|
||||
#include <array>
|
||||
#include <deque>
|
||||
#include <numeric>
|
||||
#include <set>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <test_data.hpp>
|
||||
|
||||
@@ -627,526 +624,6 @@ static void FromCborChunkedString(benchmark::State& state, const std::size_t chu
|
||||
|
||||
BENCHMARK_CAPTURE(FromCborChunkedString, 10000 chunks, 10000);
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
// SAX-parse JSON from string
|
||||
//
|
||||
// sax_parse() with a handler that does next to nothing runs the lexer and
|
||||
// parser without building values, so the gap to ParseString above is the cost
|
||||
// of building the DOM.
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
static std::string read_file(const char* filename)
|
||||
{
|
||||
std::ifstream f(filename);
|
||||
return std::string((std::istreambuf_iterator<char>(f)), std::istreambuf_iterator<char>());
|
||||
}
|
||||
|
||||
static json load_json(const char* filename)
|
||||
{
|
||||
return json::parse(read_file(filename));
|
||||
}
|
||||
|
||||
/// SAX handler that only counts the events, so that the handler costs next to nothing
|
||||
class counting_sax : public nlohmann::json_sax<json>
|
||||
{
|
||||
public:
|
||||
std::size_t events = 0;
|
||||
|
||||
bool null() override
|
||||
{
|
||||
return count();
|
||||
}
|
||||
bool boolean(bool /*val*/) override
|
||||
{
|
||||
return count();
|
||||
}
|
||||
bool number_integer(number_integer_t /*val*/) override
|
||||
{
|
||||
return count();
|
||||
}
|
||||
bool number_unsigned(number_unsigned_t /*val*/) override
|
||||
{
|
||||
return count();
|
||||
}
|
||||
bool number_float(number_float_t /*val*/, const string_t& /*s*/) override
|
||||
{
|
||||
return count();
|
||||
}
|
||||
bool string(string_t& /*val*/) override
|
||||
{
|
||||
return count();
|
||||
}
|
||||
bool binary(binary_t& /*val*/) override
|
||||
{
|
||||
return count();
|
||||
}
|
||||
bool start_object(std::size_t /*elements*/) override
|
||||
{
|
||||
return count();
|
||||
}
|
||||
bool key(string_t& /*val*/) override
|
||||
{
|
||||
return count();
|
||||
}
|
||||
bool end_object() override
|
||||
{
|
||||
return count();
|
||||
}
|
||||
bool start_array(std::size_t /*elements*/) override
|
||||
{
|
||||
return count();
|
||||
}
|
||||
bool end_array() override
|
||||
{
|
||||
return count();
|
||||
}
|
||||
bool parse_error(std::size_t /*position*/, const std::string& /*last_token*/, const json::exception& /*ex*/) override
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
private:
|
||||
bool count()
|
||||
{
|
||||
++events;
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
static void SaxParse(benchmark::State& state, const char* filename)
|
||||
{
|
||||
const std::string str = read_file(filename);
|
||||
|
||||
for (auto _ : state)
|
||||
{
|
||||
counting_sax sax;
|
||||
bool result = json::sax_parse(str, &sax);
|
||||
benchmark::DoNotOptimize(result);
|
||||
benchmark::DoNotOptimize(sax.events);
|
||||
}
|
||||
|
||||
state.SetBytesProcessed(state.iterations() * str.size());
|
||||
}
|
||||
BENCHMARK_CAPTURE(SaxParse, canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json");
|
||||
BENCHMARK_CAPTURE(SaxParse, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
|
||||
BENCHMARK_CAPTURE(SaxParse, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
// work with values: copy, destroy, compare, iterate
|
||||
//
|
||||
// The parsed test files stand in for a document an application keeps around.
|
||||
// Each benchmark times one whole-document operation; setup and cleanup run
|
||||
// outside the timed section.
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
static void Copy(benchmark::State& state, const char* filename)
|
||||
{
|
||||
const json j = load_json(filename);
|
||||
|
||||
for (auto _ : state)
|
||||
{
|
||||
auto* copy = new json(j);
|
||||
|
||||
state.PauseTiming();
|
||||
delete copy;
|
||||
state.ResumeTiming();
|
||||
}
|
||||
}
|
||||
BENCHMARK_CAPTURE(Copy, canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json");
|
||||
BENCHMARK_CAPTURE(Copy, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
|
||||
BENCHMARK_CAPTURE(Copy, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
|
||||
|
||||
static void Destroy(benchmark::State& state, const char* filename)
|
||||
{
|
||||
const json j = load_json(filename);
|
||||
|
||||
for (auto _ : state)
|
||||
{
|
||||
state.PauseTiming();
|
||||
auto* copy = new json(j);
|
||||
state.ResumeTiming();
|
||||
|
||||
delete copy;
|
||||
}
|
||||
}
|
||||
BENCHMARK_CAPTURE(Destroy, canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json");
|
||||
BENCHMARK_CAPTURE(Destroy, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
|
||||
BENCHMARK_CAPTURE(Destroy, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
|
||||
|
||||
static void Equal(benchmark::State& state, const char* filename)
|
||||
{
|
||||
// two separate values, so that the comparison cannot stop at identical addresses
|
||||
const json j = load_json(filename);
|
||||
const json copy = j;
|
||||
|
||||
for (auto _ : state)
|
||||
{
|
||||
bool result = (j == copy);
|
||||
benchmark::DoNotOptimize(result);
|
||||
}
|
||||
}
|
||||
BENCHMARK_CAPTURE(Equal, canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json");
|
||||
BENCHMARK_CAPTURE(Equal, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
|
||||
BENCHMARK_CAPTURE(Equal, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
|
||||
|
||||
/// visit every value with range-based for loops, as client code walks a document
|
||||
static std::size_t count_values(const json& j)
|
||||
{
|
||||
std::size_t result = 1;
|
||||
if (j.is_structured())
|
||||
{
|
||||
for (const auto& element : j)
|
||||
{
|
||||
result += count_values(element);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
static void Iterate(benchmark::State& state, const char* filename)
|
||||
{
|
||||
const json j = load_json(filename);
|
||||
|
||||
for (auto _ : state)
|
||||
{
|
||||
std::size_t result = count_values(j);
|
||||
benchmark::DoNotOptimize(result);
|
||||
}
|
||||
|
||||
state.SetItemsProcessed(state.iterations() * static_cast<std::int64_t>(count_values(j)));
|
||||
}
|
||||
BENCHMARK_CAPTURE(Iterate, canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json");
|
||||
BENCHMARK_CAPTURE(Iterate, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
|
||||
BENCHMARK_CAPTURE(Iterate, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
// flatten and unflatten
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
static void Flatten(benchmark::State& state, const char* filename)
|
||||
{
|
||||
const json j = load_json(filename);
|
||||
|
||||
for (auto _ : state)
|
||||
{
|
||||
json flat = j.flatten();
|
||||
benchmark::DoNotOptimize(flat);
|
||||
}
|
||||
}
|
||||
BENCHMARK_CAPTURE(Flatten, canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json");
|
||||
BENCHMARK_CAPTURE(Flatten, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
|
||||
BENCHMARK_CAPTURE(Flatten, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
|
||||
|
||||
// ordered_json objects look keys up linearly, which flatten() used to do for
|
||||
// every value; canada (330,000 values in one flat object) is left out, as it
|
||||
// would take minutes with such versions
|
||||
static void FlattenOrdered(benchmark::State& state, const char* filename)
|
||||
{
|
||||
const nlohmann::ordered_json j = nlohmann::ordered_json::parse(read_file(filename));
|
||||
|
||||
for (auto _ : state)
|
||||
{
|
||||
nlohmann::ordered_json flat = j.flatten();
|
||||
benchmark::DoNotOptimize(flat);
|
||||
}
|
||||
}
|
||||
BENCHMARK_CAPTURE(FlattenOrdered, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
|
||||
BENCHMARK_CAPTURE(FlattenOrdered, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
|
||||
|
||||
static void Unflatten(benchmark::State& state, const char* filename)
|
||||
{
|
||||
const json flat = load_json(filename).flatten();
|
||||
|
||||
for (auto _ : state)
|
||||
{
|
||||
json j = flat.unflatten();
|
||||
benchmark::DoNotOptimize(j);
|
||||
}
|
||||
}
|
||||
BENCHMARK_CAPTURE(Unflatten, canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json");
|
||||
BENCHMARK_CAPTURE(Unflatten, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
|
||||
BENCHMARK_CAPTURE(Unflatten, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
// diff and patch
|
||||
//
|
||||
// The target is the document with every eighth scalar changed, so the patch
|
||||
// has one "replace" operation per changed value and the structure stays the
|
||||
// same.
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
static void change_every_eighth_scalar(json& j, std::size_t& counter)
|
||||
{
|
||||
if (j.is_structured())
|
||||
{
|
||||
for (auto& element : j)
|
||||
{
|
||||
change_every_eighth_scalar(element, counter);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (++counter % 8 != 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
switch (j.type())
|
||||
{
|
||||
case json::value_t::number_integer:
|
||||
j = j.get<json::number_integer_t>() + 1;
|
||||
break;
|
||||
case json::value_t::number_unsigned:
|
||||
j = j.get<json::number_unsigned_t>() + 1;
|
||||
break;
|
||||
case json::value_t::number_float:
|
||||
j = j.get<json::number_float_t>() + 0.5;
|
||||
break;
|
||||
case json::value_t::string:
|
||||
j = j.get<std::string>() + "!";
|
||||
break;
|
||||
case json::value_t::boolean:
|
||||
j = !j.get<bool>();
|
||||
break;
|
||||
case json::value_t::null:
|
||||
case json::value_t::object:
|
||||
case json::value_t::array:
|
||||
case json::value_t::binary:
|
||||
case json::value_t::discarded:
|
||||
default:
|
||||
j = 0;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
static json changed_copy(const json& j)
|
||||
{
|
||||
json result = j;
|
||||
std::size_t counter = 0;
|
||||
change_every_eighth_scalar(result, counter);
|
||||
return result;
|
||||
}
|
||||
|
||||
static void Diff(benchmark::State& state, const char* filename)
|
||||
{
|
||||
const json source = load_json(filename);
|
||||
const json target = changed_copy(source);
|
||||
|
||||
for (auto _ : state)
|
||||
{
|
||||
json patch = json::diff(source, target);
|
||||
benchmark::DoNotOptimize(patch);
|
||||
}
|
||||
|
||||
state.SetItemsProcessed(state.iterations() * static_cast<std::int64_t>(json::diff(source, target).size()));
|
||||
}
|
||||
BENCHMARK_CAPTURE(Diff, canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json");
|
||||
BENCHMARK_CAPTURE(Diff, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
|
||||
BENCHMARK_CAPTURE(Diff, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
|
||||
|
||||
static void Patch(benchmark::State& state, const char* filename)
|
||||
{
|
||||
const json source = load_json(filename);
|
||||
const json patch = json::diff(source, changed_copy(source));
|
||||
|
||||
for (auto _ : state)
|
||||
{
|
||||
json result = source.patch(patch);
|
||||
benchmark::DoNotOptimize(result);
|
||||
}
|
||||
|
||||
state.SetItemsProcessed(state.iterations() * static_cast<std::int64_t>(patch.size()));
|
||||
}
|
||||
BENCHMARK_CAPTURE(Patch, canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json");
|
||||
BENCHMARK_CAPTURE(Patch, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
|
||||
BENCHMARK_CAPTURE(Patch, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
// JSON Pointer
|
||||
//
|
||||
// The pointers are those of every scalar in the document, i.e., the keys of
|
||||
// its flattened form.
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
static std::vector<std::string> pointer_strings(const json& j)
|
||||
{
|
||||
std::vector<std::string> result;
|
||||
const json flat = j.flatten();
|
||||
result.reserve(flat.size());
|
||||
for (auto it = flat.begin(); it != flat.end(); ++it)
|
||||
{
|
||||
result.push_back(it.key());
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
static std::vector<json::json_pointer> pointers(const json& j)
|
||||
{
|
||||
const std::vector<std::string> strings = pointer_strings(j);
|
||||
std::vector<json::json_pointer> result;
|
||||
result.reserve(strings.size());
|
||||
for (const auto& s : strings)
|
||||
{
|
||||
result.emplace_back(s);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
static void JsonPointerParse(benchmark::State& state, const char* filename)
|
||||
{
|
||||
const std::vector<std::string> strings = pointer_strings(load_json(filename));
|
||||
|
||||
for (auto _ : state)
|
||||
{
|
||||
for (const auto& s : strings)
|
||||
{
|
||||
json::json_pointer ptr(s);
|
||||
benchmark::DoNotOptimize(ptr);
|
||||
}
|
||||
}
|
||||
|
||||
state.SetItemsProcessed(state.iterations() * static_cast<std::int64_t>(strings.size()));
|
||||
}
|
||||
BENCHMARK_CAPTURE(JsonPointerParse, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
|
||||
BENCHMARK_CAPTURE(JsonPointerParse, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
|
||||
|
||||
static void JsonPointerToString(benchmark::State& state, const char* filename)
|
||||
{
|
||||
const std::vector<json::json_pointer> ptrs = pointers(load_json(filename));
|
||||
|
||||
for (auto _ : state)
|
||||
{
|
||||
for (const auto& ptr : ptrs)
|
||||
{
|
||||
std::string s = ptr.to_string();
|
||||
benchmark::DoNotOptimize(s);
|
||||
}
|
||||
}
|
||||
|
||||
state.SetItemsProcessed(state.iterations() * static_cast<std::int64_t>(ptrs.size()));
|
||||
}
|
||||
BENCHMARK_CAPTURE(JsonPointerToString, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
|
||||
BENCHMARK_CAPTURE(JsonPointerToString, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
|
||||
|
||||
static void JsonPointerAt(benchmark::State& state, const char* filename)
|
||||
{
|
||||
const json j = load_json(filename);
|
||||
const std::vector<json::json_pointer> ptrs = pointers(j);
|
||||
|
||||
for (auto _ : state)
|
||||
{
|
||||
for (const auto& ptr : ptrs)
|
||||
{
|
||||
const json& value = j.at(ptr);
|
||||
benchmark::DoNotOptimize(&value);
|
||||
}
|
||||
}
|
||||
|
||||
state.SetItemsProcessed(state.iterations() * static_cast<std::int64_t>(ptrs.size()));
|
||||
}
|
||||
BENCHMARK_CAPTURE(JsonPointerAt, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
|
||||
BENCHMARK_CAPTURE(JsonPointerAt, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
|
||||
|
||||
static void JsonPointerContains(benchmark::State& state, const char* filename)
|
||||
{
|
||||
const json j = load_json(filename);
|
||||
const std::vector<json::json_pointer> ptrs = pointers(j);
|
||||
|
||||
for (auto _ : state)
|
||||
{
|
||||
for (const auto& ptr : ptrs)
|
||||
{
|
||||
bool result = j.contains(ptr);
|
||||
benchmark::DoNotOptimize(result);
|
||||
}
|
||||
}
|
||||
|
||||
state.SetItemsProcessed(state.iterations() * static_cast<std::int64_t>(ptrs.size()));
|
||||
}
|
||||
BENCHMARK_CAPTURE(JsonPointerContains, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
|
||||
BENCHMARK_CAPTURE(JsonPointerContains, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
// conversions from and to C++ containers
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
static void GetVectorInt(benchmark::State& state, const char* filename)
|
||||
{
|
||||
const json j = load_json(filename);
|
||||
|
||||
for (auto _ : state)
|
||||
{
|
||||
auto v = j.get<std::vector<std::int64_t>>();
|
||||
benchmark::DoNotOptimize(v);
|
||||
}
|
||||
|
||||
state.SetItemsProcessed(state.iterations() * static_cast<std::int64_t>(j.size()));
|
||||
}
|
||||
BENCHMARK_CAPTURE(GetVectorInt, signed_ints, TEST_DATA_DIRECTORY "/regression/signed_ints.json");
|
||||
|
||||
static void GetVectorDouble(benchmark::State& state, const char* filename)
|
||||
{
|
||||
const json j = load_json(filename);
|
||||
|
||||
for (auto _ : state)
|
||||
{
|
||||
auto v = j.get<std::vector<double>>();
|
||||
benchmark::DoNotOptimize(v);
|
||||
}
|
||||
|
||||
state.SetItemsProcessed(state.iterations() * static_cast<std::int64_t>(j.size()));
|
||||
}
|
||||
BENCHMARK_CAPTURE(GetVectorDouble, floats, TEST_DATA_DIRECTORY "/regression/floats.json");
|
||||
|
||||
// a large flat object: the flattened document, one member per scalar
|
||||
static void GetMap(benchmark::State& state, const char* filename)
|
||||
{
|
||||
const json j = load_json(filename).flatten();
|
||||
|
||||
for (auto _ : state)
|
||||
{
|
||||
auto m = j.get<std::map<std::string, json>>();
|
||||
benchmark::DoNotOptimize(m);
|
||||
}
|
||||
|
||||
state.SetItemsProcessed(state.iterations() * static_cast<std::int64_t>(j.size()));
|
||||
}
|
||||
BENCHMARK_CAPTURE(GetMap, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
|
||||
|
||||
/// 100,000 strings, each too long for the small string optimization
|
||||
static std::vector<std::string> make_strings()
|
||||
{
|
||||
std::vector<std::string> result;
|
||||
result.reserve(100000);
|
||||
for (std::size_t i = 0; i < 100000; ++i)
|
||||
{
|
||||
result.push_back("a string that does not fit in place: " + std::to_string(i));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
static void ToJsonStrings(benchmark::State& state, const bool move)
|
||||
{
|
||||
const std::vector<std::string> strings = make_strings();
|
||||
|
||||
for (auto _ : state)
|
||||
{
|
||||
state.PauseTiming();
|
||||
std::vector<std::string> input = strings;
|
||||
state.ResumeTiming();
|
||||
|
||||
auto* j = move ? new json(std::move(input)) : new json(input);
|
||||
|
||||
state.PauseTiming();
|
||||
delete j;
|
||||
state.ResumeTiming();
|
||||
}
|
||||
|
||||
state.SetItemsProcessed(state.iterations() * static_cast<std::int64_t>(strings.size()));
|
||||
}
|
||||
BENCHMARK_CAPTURE(ToJsonStrings, copy, false);
|
||||
BENCHMARK_CAPTURE(ToJsonStrings, move, true);
|
||||
|
||||
// conversion of a JSON array into standard containers
|
||||
template<typename Container>
|
||||
static void GetArray(benchmark::State& state)
|
||||
|
||||
@@ -0,0 +1,474 @@
|
||||
// __ _____ _____ _____
|
||||
// __| | __| | | | JSON for Modern C++ (supporting code)
|
||||
// | | |__ | | | | | | version 3.12.0
|
||||
// |_____|_____|_____|_|___| https://github.com/nlohmann/json
|
||||
//
|
||||
// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
// The fast paths for contiguous input (bulk string scanning, SWAR searches,
|
||||
// UTF-8 prefix validation, the 8-digit number loop, the 8-byte indentation
|
||||
// skip, the BSON/BON8 bulk string readers) load several bytes at a time. This
|
||||
// test places every input directly in front of (or directly behind) an
|
||||
// inaccessible page, so reading even one byte outside the input crashes, with
|
||||
// or without sanitizers. ASan alone misses over-reads into the slack of the
|
||||
// same heap allocation.
|
||||
|
||||
#include "doctest_compatibility.h"
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
using nlohmann::json;
|
||||
|
||||
#include <cstdint>
|
||||
#include <cstring>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#if defined(_WIN32)
|
||||
#define JSON_TEST_GUARD_PAGE_WINDOWS
|
||||
#ifndef NOMINMAX
|
||||
#define NOMINMAX
|
||||
#endif
|
||||
#ifndef WIN32_LEAN_AND_MEAN
|
||||
#define WIN32_LEAN_AND_MEAN
|
||||
#endif
|
||||
#include <windows.h>
|
||||
#elif defined(__unix__) || defined(__APPLE__)
|
||||
#define JSON_TEST_GUARD_PAGE_POSIX
|
||||
#include <sys/mman.h>
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
#if defined(JSON_TEST_GUARD_PAGE_WINDOWS) || defined(JSON_TEST_GUARD_PAGE_POSIX)
|
||||
|
||||
namespace
|
||||
{
|
||||
// Three pages: an inaccessible one, a readable one, an inaccessible one. An
|
||||
// input is copied either to the end of the middle page (any read past its last
|
||||
// byte faults) or to its start (any read before its first byte faults).
|
||||
class guarded_buffer
|
||||
{
|
||||
public:
|
||||
guarded_buffer()
|
||||
: page(page_size())
|
||||
{
|
||||
#if defined(JSON_TEST_GUARD_PAGE_WINDOWS)
|
||||
void* const p = VirtualAlloc(nullptr, 3 * page, MEM_RESERVE | MEM_COMMIT, PAGE_NOACCESS);
|
||||
REQUIRE(p != nullptr);
|
||||
base = static_cast<unsigned char*>(p);
|
||||
DWORD old_protect = 0;
|
||||
REQUIRE(VirtualProtect(base + page, page, PAGE_READWRITE, &old_protect) != 0);
|
||||
#else
|
||||
void* const p = mmap(nullptr, 3 * page, PROT_NONE, MAP_PRIVATE | MAP_ANON, -1, 0);
|
||||
REQUIRE(p != MAP_FAILED);
|
||||
base = static_cast<unsigned char*>(p);
|
||||
REQUIRE(mprotect(base + page, page, PROT_READ | PROT_WRITE) == 0);
|
||||
#endif
|
||||
}
|
||||
|
||||
~guarded_buffer()
|
||||
{
|
||||
#if defined(JSON_TEST_GUARD_PAGE_WINDOWS)
|
||||
VirtualFree(base, 0, MEM_RELEASE);
|
||||
#else
|
||||
munmap(base, 3 * page);
|
||||
#endif
|
||||
}
|
||||
|
||||
guarded_buffer(const guarded_buffer&) = delete;
|
||||
guarded_buffer& operator=(const guarded_buffer&) = delete;
|
||||
guarded_buffer(guarded_buffer&&) = delete;
|
||||
guarded_buffer& operator=(guarded_buffer&&) = delete;
|
||||
|
||||
// copies @a input so its last byte is the last readable byte
|
||||
const std::uint8_t* at_end(const std::string& input)
|
||||
{
|
||||
REQUIRE(input.size() <= page);
|
||||
unsigned char* const first = base + (2 * page) - input.size();
|
||||
std::memcpy(first, input.data(), input.size());
|
||||
return first;
|
||||
}
|
||||
|
||||
// copies @a input so its first byte is the first readable byte
|
||||
const std::uint8_t* at_start(const std::string& input)
|
||||
{
|
||||
REQUIRE(input.size() <= page);
|
||||
unsigned char* const first = base + page;
|
||||
std::memcpy(first, input.data(), input.size());
|
||||
return first;
|
||||
}
|
||||
|
||||
private:
|
||||
static std::size_t page_size()
|
||||
{
|
||||
#if defined(JSON_TEST_GUARD_PAGE_WINDOWS)
|
||||
SYSTEM_INFO info;
|
||||
GetSystemInfo(&info);
|
||||
return static_cast<std::size_t>(info.dwPageSize);
|
||||
#else
|
||||
return static_cast<std::size_t>(sysconf(_SC_PAGESIZE));
|
||||
#endif
|
||||
}
|
||||
|
||||
std::size_t page = 0;
|
||||
unsigned char* base = nullptr;
|
||||
};
|
||||
|
||||
// the serialization of the result of @a f, or the exception message; @a f
|
||||
// takes the allow_exceptions flag, which is false without exceptions, where
|
||||
// an error yields a discarded value instead
|
||||
template<typename F>
|
||||
std::string outcome(F f)
|
||||
{
|
||||
#if defined(JSON_NOEXCEPTION)
|
||||
return "value: " + f(false).dump(-1, ' ', false, json::error_handler_t::keep);
|
||||
#else
|
||||
try
|
||||
{
|
||||
return "value: " + f(true).dump(-1, ' ', false, json::error_handler_t::keep);
|
||||
}
|
||||
catch (const json::exception& e)
|
||||
{
|
||||
return std::string("error: ") + e.what();
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
// a SAX handler that only counts events, so sax_parse runs without a DOM
|
||||
struct counting_sax : nlohmann::json_sax<json>
|
||||
{
|
||||
std::size_t events = 0;
|
||||
std::string error{}; // NOLINT(readability-redundant-member-init)
|
||||
|
||||
bool null() override
|
||||
{
|
||||
return count();
|
||||
}
|
||||
bool boolean(bool /*val*/) override
|
||||
{
|
||||
return count();
|
||||
}
|
||||
bool number_integer(number_integer_t /*val*/) override
|
||||
{
|
||||
return count();
|
||||
}
|
||||
bool number_unsigned(number_unsigned_t /*val*/) override
|
||||
{
|
||||
return count();
|
||||
}
|
||||
bool number_float(number_float_t /*val*/, const string_t& /*s*/) override
|
||||
{
|
||||
return count();
|
||||
}
|
||||
bool string(string_t& /*val*/) override
|
||||
{
|
||||
return count();
|
||||
}
|
||||
bool binary(binary_t& /*val*/) override
|
||||
{
|
||||
return count();
|
||||
}
|
||||
bool start_object(std::size_t /*elements*/) override
|
||||
{
|
||||
return count();
|
||||
}
|
||||
bool key(string_t& /*val*/) override
|
||||
{
|
||||
return count();
|
||||
}
|
||||
bool end_object() override
|
||||
{
|
||||
return count();
|
||||
}
|
||||
bool start_array(std::size_t /*elements*/) override
|
||||
{
|
||||
return count();
|
||||
}
|
||||
bool end_array() override
|
||||
{
|
||||
return count();
|
||||
}
|
||||
bool parse_error(std::size_t position, const std::string& /*last_token*/, const nlohmann::detail::exception& ex) override
|
||||
{
|
||||
error = std::to_string(position) + " " + ex.what();
|
||||
return false;
|
||||
}
|
||||
|
||||
private:
|
||||
bool count()
|
||||
{
|
||||
++events;
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
// all prefixes of each seed, so the input ends at every byte of it
|
||||
std::vector<std::string> all_prefixes(const std::vector<std::string>& seeds)
|
||||
{
|
||||
std::vector<std::string> result;
|
||||
for (const auto& seed : seeds)
|
||||
{
|
||||
for (std::size_t n = 0; n <= seed.size(); ++n)
|
||||
{
|
||||
result.push_back(seed.substr(0, n));
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
std::vector<std::string> json_seeds()
|
||||
{
|
||||
std::vector<std::string> seeds =
|
||||
{
|
||||
// string runs: ASCII, escapes, \u escapes, surrogate pairs
|
||||
"\"abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ\"",
|
||||
R"("ab\ncd\"ef\\gh\u00e4ij\uD83D\uDE00kl\u12")",
|
||||
// 2-, 3- and 4-byte UTF-8 sequences, then invalid UTF-8
|
||||
std::string("\"\xC3\xA4\xE2\x82\xAC\xF0\x9F\x98\x80\xC3\xA4\xE2\x82\xAC\xF0\x9F\x98\x80") + "abcdefgh\"",
|
||||
"\"abcdefgh\xC3\x28 \xED\xA0\x80 \xF4\x90\x80\x80 \xF8 \xC0\xAF\"",
|
||||
std::string("\"abcdefg\x01") + "abcdefgh\"",
|
||||
// numbers, so the 8-digit loop runs up to the end
|
||||
"12345678",
|
||||
"1234567890123456",
|
||||
"12345678901234567890123",
|
||||
"-0.000000001234567890123456789",
|
||||
"3.14159265358979323846264338327950288e+100",
|
||||
"-1.2345678901234567e-12",
|
||||
"1E-5",
|
||||
"[1.5,2e10,-3E-5,0.000000001,123456789012345678]",
|
||||
// literals
|
||||
"[true,false,null]",
|
||||
// comments (only parsed with ignore_comments)
|
||||
"/* comment */ [1, // line comment\n 2] /**/",
|
||||
// a pretty-printed document
|
||||
json::parse(R"({"a": [1, 2.5, "three", {"b": null, "c": [true, false]}], "d": {"e": {"f": {"g": "h"}}}})").dump(4),
|
||||
json::parse(R"({"a": [1, 2.5, "three", {"b": null, "c": [true, false]}], "d": {"e": {"f": {"g": "h"}}}})").dump(8, '\t'),
|
||||
};
|
||||
|
||||
// indentation runs of every length, with and without a closing bracket
|
||||
for (std::size_t n = 0; n <= 20; ++n)
|
||||
{
|
||||
seeds.push_back("[\n" + std::string(n, ' ') + "1,\n" + std::string(n, ' ') + "2\n]");
|
||||
}
|
||||
|
||||
return all_prefixes(seeds);
|
||||
}
|
||||
|
||||
// a value whose binary serializations contain long strings and keys, numbers
|
||||
// of every width, and nested containers
|
||||
json binary_document()
|
||||
{
|
||||
json j;
|
||||
j["a key that is longer than eight bytes"] = "a string value that is longer than eight bytes";
|
||||
j["\xC3\xA4\xE2\x82\xAC\xF0\x9F\x98\x80"] = "\xF0\x9F\x98\x80\xE2\x82\xAC\xC3\xA4 utf-8 0123456789";
|
||||
j["numbers"] = {0, 1, -1, 255, -256, 65536, -2147483649LL, 4294967296ULL, 1.5, -2.25e100};
|
||||
j["nested"] = {{"x", {{"y", {true, false, nullptr}}}}};
|
||||
j["s"] = "";
|
||||
return j;
|
||||
}
|
||||
|
||||
// readers for check_binary(), callable with a container or an iterator range
|
||||
// followed by the allow_exceptions flag
|
||||
#define JSON_TEST_GUARD_PAGE_READER(name, from) \
|
||||
struct name \
|
||||
{ \
|
||||
template<typename... Args> \
|
||||
json operator()(bool allow_exceptions, Args&& ... args) const \
|
||||
{ \
|
||||
return json::from(std::forward<Args>(args)..., true, allow_exceptions); \
|
||||
} \
|
||||
};
|
||||
JSON_TEST_GUARD_PAGE_READER(read_bson, from_bson)
|
||||
JSON_TEST_GUARD_PAGE_READER(read_bon8, from_bon8)
|
||||
JSON_TEST_GUARD_PAGE_READER(read_cbor, from_cbor)
|
||||
JSON_TEST_GUARD_PAGE_READER(read_msgpack, from_msgpack)
|
||||
JSON_TEST_GUARD_PAGE_READER(read_ubjson, from_ubjson)
|
||||
JSON_TEST_GUARD_PAGE_READER(read_bjdata, from_bjdata)
|
||||
#undef JSON_TEST_GUARD_PAGE_READER
|
||||
|
||||
// reads every prefix of @a bytes at both ends of the guarded page and from a
|
||||
// std::string, which must give the same outcome
|
||||
template<typename Reader>
|
||||
void check_binary(guarded_buffer& guard, const std::vector<std::uint8_t>& bytes, Reader parse)
|
||||
{
|
||||
for (const auto& input : all_prefixes({std::string(bytes.begin(), bytes.end())}))
|
||||
{
|
||||
CAPTURE(input)
|
||||
const std::string expected = outcome([&](bool allow)
|
||||
{
|
||||
return parse(allow, input);
|
||||
});
|
||||
for (int placement = 0; placement < 2; ++placement)
|
||||
{
|
||||
const std::uint8_t* const first = placement == 0 ? guard.at_end(input) : guard.at_start(input);
|
||||
CHECK(outcome([&](bool allow)
|
||||
{
|
||||
return parse(allow, first, first + input.size());
|
||||
}) == expected);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// parse_float_native() at both ends of the guarded page, compared bit for bit
|
||||
// with converting the same token from a std::string
|
||||
template<typename FloatType>
|
||||
void check_float_native(guarded_buffer& guard, const std::string& token)
|
||||
{
|
||||
const std::size_t decimal_point = token.find('.');
|
||||
const std::size_t exponent = token.find_first_of("eE");
|
||||
const std::size_t mantissa_end = exponent == std::string::npos ? token.size() : exponent;
|
||||
const FloatType expected = nlohmann::detail::parse_float_native<FloatType>(token.data(), token.data() + token.size(), decimal_point, mantissa_end);
|
||||
for (int placement = 0; placement < 2; ++placement)
|
||||
{
|
||||
const char* const first = reinterpret_cast<const char*>(placement == 0 ? guard.at_end(token) : guard.at_start(token));
|
||||
const FloatType value = nlohmann::detail::parse_float_native<FloatType>(first, first + token.size(), decimal_point, mantissa_end);
|
||||
CHECK(std::memcmp(&value, &expected, sizeof(value)) == 0);
|
||||
}
|
||||
}
|
||||
} // namespace
|
||||
|
||||
TEST_CASE("guard page")
|
||||
{
|
||||
guarded_buffer guard;
|
||||
|
||||
SECTION("string scanning")
|
||||
{
|
||||
// fills that drive each function to a different stop
|
||||
const std::vector<std::string> fills =
|
||||
{
|
||||
std::string(64, 'a'),
|
||||
std::string(64, '"'),
|
||||
std::string(64, '\\'),
|
||||
std::string(64, '\x01'),
|
||||
std::string(64, '\x7F'),
|
||||
std::string(64, '\xFF'),
|
||||
};
|
||||
std::vector<std::string> patterns = fills;
|
||||
{
|
||||
std::string utf8;
|
||||
while (utf8.size() < 64)
|
||||
{
|
||||
utf8 += "\xC3\xA4\xE2\x82\xAC\xF0\x9F\x98\x80" "ab";
|
||||
}
|
||||
patterns.push_back(utf8);
|
||||
}
|
||||
{
|
||||
std::string bytes;
|
||||
for (unsigned int i = 0; i < 64; ++i)
|
||||
{
|
||||
bytes.push_back(static_cast<char>((i * 37u) & 0xFFu));
|
||||
}
|
||||
patterns.push_back(bytes);
|
||||
}
|
||||
|
||||
for (const auto& pattern : patterns)
|
||||
{
|
||||
for (std::size_t n = 0; n <= 64; ++n)
|
||||
{
|
||||
const std::string input = pattern.substr(0, n);
|
||||
for (int placement = 0; placement < 2; ++placement)
|
||||
{
|
||||
const unsigned char* const p = placement == 0 ? guard.at_end(input) : guard.at_start(input);
|
||||
|
||||
CHECK(nlohmann::detail::find_string_special(p, n) <= n);
|
||||
CHECK(nlohmann::detail::find_ascii_copyable_run(p, n) <= n);
|
||||
CHECK(nlohmann::detail::valid_utf8_prefix(p, n) <= n);
|
||||
#if defined(JSON_USE_SIMDUTF) && defined(JSON_HAS_CPP_17)
|
||||
CHECK(nlohmann::detail::find_string_delimiter(p, n) <= n);
|
||||
#endif
|
||||
CHECK(nlohmann::detail::string_bulk_run(p, n) <= n);
|
||||
CHECK(nlohmann::detail::scalar_string_bulk_run(p, n) <= n);
|
||||
for (std::size_t i = 0; i < n; ++i)
|
||||
{
|
||||
CHECK(nlohmann::detail::validate_one_utf8(p + i, n - i) <= n - i);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
SECTION("float conversion")
|
||||
{
|
||||
// the lexer converts floats from its token buffer, so the JSON section
|
||||
// below never places a float token at the page boundary; this calls the
|
||||
// conversion directly: the 8-digit loops of parse_float_significand()
|
||||
// and digit_comparison() read up to the end of the token
|
||||
const std::vector<std::string> tokens = all_prefixes(
|
||||
{
|
||||
"1234567812345678123456781234567812345678",
|
||||
"-1.2345678901234567890123456789e-123",
|
||||
"0.00000000000000000000000000000000000001",
|
||||
"98765432.10987654",
|
||||
"123456789012345678901234567890e-30",
|
||||
// halfway between 1 and the next double, then just above it
|
||||
"1.00000000000000011102230246251565404236316680908203125",
|
||||
"1.000000000000000111022302462515654042363166809082031250000000001",
|
||||
});
|
||||
for (const auto& token : tokens)
|
||||
{
|
||||
// only complete number tokens: the conversion expects validated input
|
||||
if (!token.empty() && json::accept(token))
|
||||
{
|
||||
CAPTURE(token)
|
||||
check_float_native<double>(guard, token);
|
||||
check_float_native<float>(guard, token);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
SECTION("JSON")
|
||||
{
|
||||
for (const auto& input : json_seeds())
|
||||
{
|
||||
CAPTURE(input)
|
||||
const std::string expected = outcome([&](bool allow)
|
||||
{
|
||||
return json::parse(input, nullptr, allow);
|
||||
});
|
||||
const std::string expected_comments = outcome([&](bool allow)
|
||||
{
|
||||
return json::parse(input, nullptr, allow, true);
|
||||
});
|
||||
const bool expected_accept = json::accept(input);
|
||||
counting_sax expected_sax;
|
||||
const bool expected_sax_result = json::sax_parse(input, &expected_sax);
|
||||
|
||||
for (int placement = 0; placement < 2; ++placement)
|
||||
{
|
||||
const std::uint8_t* const first = placement == 0 ? guard.at_end(input) : guard.at_start(input);
|
||||
const std::uint8_t* const last = first + input.size();
|
||||
|
||||
CHECK(outcome([&](bool allow)
|
||||
{
|
||||
return json::parse(first, last, nullptr, allow);
|
||||
}) == expected);
|
||||
CHECK(outcome([&](bool allow)
|
||||
{
|
||||
return json::parse(first, last, nullptr, allow, true);
|
||||
}) == expected_comments);
|
||||
CHECK(json::accept(first, last) == expected_accept);
|
||||
|
||||
counting_sax sax;
|
||||
CHECK(json::sax_parse(first, last, &sax) == expected_sax_result);
|
||||
CHECK(sax.events == expected_sax.events);
|
||||
CHECK(sax.error == expected_sax.error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
SECTION("binary formats")
|
||||
{
|
||||
// every prefix ends inside a value, e.g., inside a BSON key or BON8
|
||||
// string without its terminator
|
||||
const json j = binary_document();
|
||||
check_binary(guard, json::to_bson(j), read_bson());
|
||||
check_binary(guard, json::to_bon8(j), read_bon8());
|
||||
check_binary(guard, json::to_cbor(j), read_cbor());
|
||||
check_binary(guard, json::to_msgpack(j), read_msgpack());
|
||||
check_binary(guard, json::to_ubjson(j), read_ubjson());
|
||||
check_binary(guard, json::to_bjdata(j), read_bjdata());
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
Reference in new issue
Block a user