mirror of
https://github.com/nlohmann/json.git
synced 2026-10-06 14:40:32 +00:00
Add json_document and json_view: node index, parser, and document
Add json_document and json_view, a read-only, zero-copy index of a JSON text, as the first public slice of the zero-copy view (#5295). A parse produces a flat array of 16-byte nodes in document order, one per value and one per object key. Strings stay in the source text; escaped strings are decoded into an arena. Integers are converted while their digits are in the cache; floats keep only their digit layout and are converted on read. Containers store the size of their subtree, so a reader can step over one in constant time. A document makes a handful of allocations, however many values it has. The parser accepts exactly what json::parse accepts, with every combination of ignore_comments and ignore_trailing_commas, with and without a trailing NUL, and under JSON_STRICT_NUL_HANDLING. It is portable C++11 and does not depend on byte order. basic_json_document adds parse, parse_copy, accept, read (reuses a document's memory), root, is_discarded, source, owns_source, node_count, memory_usage, and shrink_to_fit. basic_json_view adds type, the is_* queries, operator bool, size, empty, materialize, and source_offset. A parse error throws the same exception basic_json::parse would throw for the same input, message and position included. detail::abi_config keeps JSON_STRICT_NUL_HANDLING and JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON readable after json.hpp undefines them, in the ABI namespace so they always match the basic_json in use. A NUL byte that ends a // comment is the end of the input, as in parse() since #5696. Signed-off-by: Niels Lohmann <mail@nlohmann.me>
This commit is contained in:
Niels Lohmann committed
1 parent
17842694f6
commit
4303ba674d
146 files changed
+9611
-13
No files matched your search
@@ -0,0 +1,100 @@
|
||||
// __ _____ _____ _____
|
||||
// __| | __| | | | JSON for Modern C++ (supporting code)
|
||||
// | | |__ | | | | | | version 3.12.0
|
||||
// |_____|_____|_____|_|___| https://github.com/nlohmann/json
|
||||
//
|
||||
// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
/*
|
||||
This file implements a parser test suitable for fuzz testing. It checks that
|
||||
json_document (the zero-copy, read-only view of a parsed JSON text declared in
|
||||
json_view.hpp) agrees with basic_json on every input:
|
||||
|
||||
- json_document::accept(data) must equal json::accept(data)
|
||||
- if the input is accepted, json_document::parse(data).root().materialize()
|
||||
must equal json::parse(data)
|
||||
- if the input is rejected, json_document::parse(data) (with exceptions
|
||||
enabled) must throw a json::parse_error or json::out_of_range whose what()
|
||||
is identical to the one json::parse(data) throws
|
||||
|
||||
The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer
|
||||
drivers.
|
||||
*/
|
||||
|
||||
#include <cassert>
|
||||
#include <string>
|
||||
#include <nlohmann/json.hpp>
|
||||
#include <nlohmann/json_view.hpp>
|
||||
|
||||
// the checks below are assertions; NDEBUG would compile them away
|
||||
#ifdef NDEBUG
|
||||
#error "the fuzzer drivers must be built without NDEBUG"
|
||||
#endif
|
||||
|
||||
using json = nlohmann::json;
|
||||
using json_document = nlohmann::json_document;
|
||||
|
||||
// see http://llvm.org/docs/LibFuzzer.html
|
||||
extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
|
||||
{
|
||||
// json_document::accept only has a single-argument overload; wrap the raw
|
||||
// bytes in a (borrowed) std::string so the same bytes can be handed to it
|
||||
const std::string input(reinterpret_cast<const char*>(data), size); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast)
|
||||
|
||||
const bool accepted_by_json = json::accept(data, data + size);
|
||||
const bool accepted_by_view = json_document::accept(input);
|
||||
|
||||
// json_document::accept must agree with json::accept on every input
|
||||
assert(accepted_by_json == accepted_by_view);
|
||||
|
||||
if (accepted_by_json)
|
||||
{
|
||||
// both parsers must agree on the resulting value
|
||||
json const j1 = json::parse(data, data + size);
|
||||
json_document const doc = json_document::parse(input);
|
||||
json const j2 = doc.root().materialize();
|
||||
assert(j1 == j2);
|
||||
}
|
||||
else
|
||||
{
|
||||
// both parsers must reject the input the same way when exceptions are used
|
||||
std::string expected_what;
|
||||
bool json_threw = false;
|
||||
try
|
||||
{
|
||||
static_cast<void>(json::parse(data, data + size));
|
||||
}
|
||||
catch (const json::parse_error& e)
|
||||
{
|
||||
expected_what = e.what();
|
||||
json_threw = true;
|
||||
}
|
||||
catch (const json::out_of_range& e)
|
||||
{
|
||||
expected_what = e.what();
|
||||
json_threw = true;
|
||||
}
|
||||
assert(json_threw);
|
||||
|
||||
bool view_threw = false;
|
||||
try
|
||||
{
|
||||
static_cast<void>(json_document::parse(input));
|
||||
}
|
||||
catch (const json::parse_error& e)
|
||||
{
|
||||
assert(e.what() == expected_what);
|
||||
view_threw = true;
|
||||
}
|
||||
catch (const json::out_of_range& e)
|
||||
{
|
||||
assert(e.what() == expected_what);
|
||||
view_threw = true;
|
||||
}
|
||||
assert(view_threw);
|
||||
}
|
||||
|
||||
// return 0 - non-zero return values are reserved for future use
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,413 @@
|
||||
// __ _____ _____ _____
|
||||
// __| | __| | | | JSON for Modern C++ (supporting code)
|
||||
// | | |__ | | | | | | version 3.12.0
|
||||
// |_____|_____|_____|_|___| https://github.com/nlohmann/json
|
||||
//
|
||||
// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
#include "doctest_compatibility.h"
|
||||
|
||||
#include <nlohmann/json_view.hpp>
|
||||
using nlohmann::json;
|
||||
using nlohmann::ordered_json;
|
||||
using nlohmann::json_document;
|
||||
using nlohmann::json_view;
|
||||
using nlohmann::ordered_json_document;
|
||||
|
||||
#include <cstdint>
|
||||
#include <list>
|
||||
#include <map>
|
||||
#include <random>
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#ifdef JSON_HAS_CPP_17
|
||||
#include <string_view>
|
||||
#endif
|
||||
|
||||
namespace
|
||||
{
|
||||
#if !defined(JSON_NOEXCEPTION)
|
||||
// the exception parse() throws for a text, or "" if it accepts it
|
||||
std::string parse_exception(const std::string& text, bool comments = false, bool trailing_commas = false)
|
||||
{
|
||||
try
|
||||
{
|
||||
const json j = json::parse(text, nullptr, true, comments, trailing_commas);
|
||||
static_cast<void>(j);
|
||||
}
|
||||
catch (const json::exception& e)
|
||||
{
|
||||
return e.what();
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
std::string view_exception(const std::string& text, bool comments = false, bool trailing_commas = false)
|
||||
{
|
||||
try
|
||||
{
|
||||
const json_document d = json_document::parse(text, true, comments, trailing_commas);
|
||||
static_cast<void>(d);
|
||||
}
|
||||
catch (const json::exception& e)
|
||||
{
|
||||
return e.what();
|
||||
}
|
||||
return "";
|
||||
}
|
||||
#endif
|
||||
|
||||
// a small deterministic generator of documents
|
||||
struct generator
|
||||
{
|
||||
std::mt19937 rng{5295}; // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed)
|
||||
|
||||
int r(int n)
|
||||
{
|
||||
return static_cast<int>(rng() % static_cast<unsigned>(n));
|
||||
}
|
||||
|
||||
void str(std::string& o)
|
||||
{
|
||||
static const char* const pieces[] = {"a", "Z", " ", "\\n", "\\\"", "\\u00e9", "\\ud83d\\ude00", "\xc3\xa9", "\xe3\x81\x82", "long text beyond the first sixteen bytes"}; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays)
|
||||
o += '"';
|
||||
for (int n = r(5); n > 0; --n)
|
||||
{
|
||||
o += pieces[r(10)];
|
||||
}
|
||||
o += '"';
|
||||
}
|
||||
|
||||
void value(std::string& o, int depth)
|
||||
{
|
||||
static const char* const scalars[] = {"0", "-1", "123456789012", "18446744073709551615", "18446744073709551616", "-9223372036854775809", // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays)
|
||||
"1.5", "-2.25e-3", "1E2", "0.1", "true", "false", "null"
|
||||
};
|
||||
const int k = depth > 5 ? 2 + r(4) : r(6);
|
||||
if (k < 2)
|
||||
{
|
||||
const bool object = k == 0;
|
||||
o += object ? '{' : '[';
|
||||
for (int i = r(5); i > 0; --i)
|
||||
{
|
||||
if (object)
|
||||
{
|
||||
str(o);
|
||||
o += r(2) == 0 ? ":" : " : ";
|
||||
}
|
||||
value(o, depth + 1);
|
||||
o += i > 1 ? ", " : "";
|
||||
}
|
||||
o += object ? '}' : ']';
|
||||
}
|
||||
else if (k < 4)
|
||||
{
|
||||
str(o);
|
||||
}
|
||||
else
|
||||
{
|
||||
o += scalars[r(13)];
|
||||
}
|
||||
}
|
||||
};
|
||||
} // namespace
|
||||
|
||||
TEST_CASE("json_view")
|
||||
{
|
||||
SECTION("types and capacity")
|
||||
{
|
||||
for (const char* text :
|
||||
{"null", "true", "false", "0", "-1", "18446744073709551615", "-9223372036854775808", "18446744073709551616", "1.5",
|
||||
"\"\"", "\"text\"", "[]", "[1,2,3]", "{}", "{\"a\":1,\"b\":2}" // NOLINT(modernize-raw-string-literal)
|
||||
})
|
||||
{
|
||||
CAPTURE(text)
|
||||
const json j = json::parse(text);
|
||||
const json_document d = json_document::parse(text);
|
||||
const json_view v = d.root();
|
||||
CHECK(v.type() == j.type());
|
||||
CHECK(v.is_null() == j.is_null());
|
||||
CHECK(v.is_boolean() == j.is_boolean());
|
||||
CHECK(v.is_number() == j.is_number());
|
||||
CHECK(v.is_number_integer() == j.is_number_integer());
|
||||
CHECK(v.is_number_unsigned() == j.is_number_unsigned());
|
||||
CHECK(v.is_number_float() == j.is_number_float());
|
||||
CHECK(v.is_string() == j.is_string());
|
||||
CHECK(v.is_array() == j.is_array());
|
||||
CHECK(v.is_object() == j.is_object());
|
||||
CHECK(v.is_binary() == j.is_binary());
|
||||
CHECK(v.is_primitive() == j.is_primitive());
|
||||
CHECK(v.is_structured() == j.is_structured());
|
||||
CHECK(!v.is_discarded());
|
||||
CHECK(static_cast<bool>(v));
|
||||
CHECK(v.size() == j.size());
|
||||
CHECK(v.empty() == j.empty());
|
||||
CHECK(v.materialize() == j);
|
||||
}
|
||||
|
||||
const json_view invalid{};
|
||||
CHECK(invalid.is_discarded());
|
||||
CHECK(!static_cast<bool>(invalid));
|
||||
CHECK(invalid.type() == json::value_t::discarded);
|
||||
CHECK(invalid.size() == 0);
|
||||
CHECK(invalid.empty());
|
||||
CHECK(invalid.materialize().is_discarded());
|
||||
CHECK(invalid.source_offset() == static_cast<std::size_t>(-1));
|
||||
}
|
||||
|
||||
SECTION("materialize")
|
||||
{
|
||||
generator g;
|
||||
for (int i = 0; i < 2000; ++i)
|
||||
{
|
||||
std::string text;
|
||||
g.value(text, 0);
|
||||
CAPTURE(text)
|
||||
CHECK(json_document::parse(text).root().materialize() == json::parse(text));
|
||||
// member order as ordered_json::parse keeps it
|
||||
CHECK(ordered_json_document::parse(text).root().materialize().dump() == ordered_json::parse(text).dump());
|
||||
}
|
||||
// duplicate keys: the last value, at the position of the first key
|
||||
CHECK(json_document::parse(R"({"a":1,"b":2,"a":3})").root().materialize() == json::parse(R"({"a":1,"b":2,"a":3})"));
|
||||
CHECK(ordered_json_document::parse(R"({"a":1,"b":2,"a":3})").root().materialize().dump() == R"({"a":3,"b":2})");
|
||||
// very deep nesting (iterative, as parse())
|
||||
const std::string deep = std::string(100000, '[') + std::string(100000, ']');
|
||||
CHECK(json_document::parse(deep).root().materialize() == json::parse(deep));
|
||||
#if JSON_DIAGNOSTICS
|
||||
// the parents are set, so errors name the path
|
||||
const json m = json_document::parse(R"({"a":{"b":[1]}})").root().materialize();
|
||||
CHECK_THROWS_WITH_AS(m.at("a").at("b").at(0).at("x"), "[json.exception.type_error.304] (/a/b/0) cannot use at() with number", json::type_error&);
|
||||
#endif
|
||||
}
|
||||
|
||||
SECTION("parse errors are those of parse()")
|
||||
{
|
||||
for (const char* text :
|
||||
{
|
||||
"", " ", "[", "]", "{", "[1,]", "{\"a\":1,}", "[1 2]", "{\"a\" 1}", "{1:2}", "tru", "nul", "fals", "truex", "-", "01", "1.", ".5", "1e",
|
||||
"\"", "\"abc", "\"\\x\"", "\"\\u12\"", "\"\\ud800\"", "\"\\udc00\"", "\"\x01\"", "\"\xff\"", "\"\xc3\"", "[1]x", "/", "/*", "[\n 1,\n x\n]", // NOLINT(modernize-raw-string-literal)
|
||||
"1e400", "-1e400", "[1.7976931348623159e308]", "{\"a\":\n{\"b\": [1, 2,\n 3 x]}}"
|
||||
})
|
||||
{
|
||||
CAPTURE(text)
|
||||
#if !defined(JSON_NOEXCEPTION)
|
||||
const std::string expected = parse_exception(text);
|
||||
REQUIRE(!expected.empty());
|
||||
CHECK(view_exception(text) == expected);
|
||||
#endif
|
||||
CHECK(!json_document::accept(text));
|
||||
const json_document d = json_document::parse(text, false);
|
||||
CHECK(d.is_discarded());
|
||||
CHECK(d.root().is_discarded());
|
||||
CHECK(d.node_count() == 0);
|
||||
}
|
||||
// the exception types
|
||||
json_document _;
|
||||
CHECK_THROWS_AS(_ = json_document::parse("[1,"), json::parse_error&);
|
||||
CHECK_THROWS_AS(_ = json_document::parse("1e400"), json::out_of_range&);
|
||||
}
|
||||
|
||||
SECTION("parse options")
|
||||
{
|
||||
for (const char* text :
|
||||
{"// c\n[1]", "[1, /* c */ 2]", "[1,]", "{\"a\":1,}", "[1,/* c */]", "/", "/* ", "[1,,]"
|
||||
})
|
||||
{
|
||||
CAPTURE(text)
|
||||
for (int options = 0; options < 4; ++options)
|
||||
{
|
||||
const bool comments = (options & 1) != 0;
|
||||
const bool trailing_commas = (options & 2) != 0;
|
||||
CHECK(json_document::accept(text, comments, trailing_commas) == json::accept(text, comments, trailing_commas));
|
||||
#if !defined(JSON_NOEXCEPTION)
|
||||
CHECK(view_exception(text, comments, trailing_commas) == parse_exception(text, comments, trailing_commas));
|
||||
#endif
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
SECTION("overflow of the floating-point type")
|
||||
{
|
||||
// the overflow is that of the document's number_float_t: 1e39
|
||||
// overflows a float, the largest float 3.4028235e38 does not, but
|
||||
// 3.4028236e38 rounds beyond it; a double document is not affected
|
||||
using json_float = nlohmann::basic_json<std::map, std::vector, std::string, bool, std::int64_t, std::uint64_t, float>;
|
||||
using float_document = nlohmann::basic_json_document<json_float>;
|
||||
CHECK_FALSE(json_float::accept("1e39"));
|
||||
CHECK(json_float::accept("3.4028235e38"));
|
||||
CHECK_FALSE(json_float::accept("3.4028236e38"));
|
||||
for (const char* text :
|
||||
{
|
||||
"1e39", "-1e39", "[3.4028235e38]", "[3.4028236e38]", "{\"a\": [1e38, -3.4028235e38]}", "{\"a\": [1e-50, 1e39]}"
|
||||
})
|
||||
{
|
||||
CAPTURE(text)
|
||||
const bool accepted = json_float::accept(text);
|
||||
CHECK(float_document::accept(text) == accepted);
|
||||
CHECK(float_document::parse(text, false).is_discarded() == !accepted);
|
||||
CHECK(json_document::accept(text));
|
||||
if (accepted)
|
||||
{
|
||||
CHECK(float_document::parse(text).root().materialize() == json_float::parse(text));
|
||||
}
|
||||
}
|
||||
float_document f;
|
||||
CHECK_THROWS_WITH_AS(f = float_document::parse("1e39"), "[json.exception.out_of_range.406] number overflow parsing '1e39'", json::out_of_range&);
|
||||
CHECK_THROWS_WITH_AS(f = float_document::parse("[3.4028236e38]"), "[json.exception.out_of_range.406] number overflow parsing '3.4028236e38'", json::out_of_range&);
|
||||
}
|
||||
|
||||
SECTION("NUL and BOM")
|
||||
{
|
||||
const std::string with_nul("[1]\0garbage", 11);
|
||||
CHECK(json_document::accept(with_nul) == json::accept(with_nul));
|
||||
const std::string nul_in_comment("[1, // c\0\n2]", 12);
|
||||
CHECK(json_document::accept(nul_in_comment, true) == json::accept(nul_in_comment, true));
|
||||
CHECK(json_document::parse("\xEF\xBB\xBF[1]").root().materialize() == json::parse("\xEF\xBB\xBF[1]"));
|
||||
#if !defined(JSON_NOEXCEPTION)
|
||||
CHECK(view_exception("\xEF\xBB") == parse_exception("\xEF\xBB"));
|
||||
#endif
|
||||
}
|
||||
|
||||
SECTION("inputs")
|
||||
{
|
||||
const std::string text = R"([1, "two", {"three": 3.5}])";
|
||||
const json expected = json::parse(text);
|
||||
|
||||
// borrowed: the text must outlive the document
|
||||
const json_document borrowed = json_document::parse(text);
|
||||
CHECK(!borrowed.owns_source());
|
||||
CHECK(borrowed.source().data() == text.data());
|
||||
CHECK(borrowed.root().materialize() == expected);
|
||||
CHECK(json_document::parse(text.c_str()).root().materialize() == expected);
|
||||
CHECK(json_document::parse(R"([1, "two", {"three": 3.5}])").root().materialize() == expected);
|
||||
CHECK(json_document::parse(text.data(), text.data() + text.size()).root().materialize() == expected);
|
||||
const std::vector<char> chars(text.begin(), text.end());
|
||||
CHECK(!json_document::parse(chars).owns_source());
|
||||
CHECK(json_document::parse(chars).root().materialize() == expected);
|
||||
const std::vector<std::uint8_t> bytes(text.begin(), text.end());
|
||||
CHECK(json_document::parse(bytes).root().materialize() == expected);
|
||||
#ifdef JSON_HAS_CPP_17
|
||||
const std::string_view sv = text;
|
||||
CHECK(!json_document::parse(sv).owns_source());
|
||||
CHECK(json_document::parse(sv).root().materialize() == expected);
|
||||
#endif
|
||||
|
||||
// owned
|
||||
std::string moved = text;
|
||||
const json_document from_rvalue = json_document::parse(std::move(moved));
|
||||
CHECK(from_rvalue.owns_source());
|
||||
CHECK(from_rvalue.root().materialize() == expected);
|
||||
CHECK(json_document::parse(std::vector<char>(text.begin(), text.end())).owns_source());
|
||||
CHECK(json_document::parse_copy(text).owns_source());
|
||||
CHECK(json_document::parse_copy(text).root().materialize() == expected);
|
||||
std::istringstream stream(text);
|
||||
const json_document from_stream = json_document::parse(stream);
|
||||
CHECK(from_stream.owns_source());
|
||||
CHECK(from_stream.root().materialize() == expected);
|
||||
const std::list<char> list(text.begin(), text.end());
|
||||
CHECK(json_document::parse(list.begin(), list.end()).owns_source());
|
||||
CHECK(json_document::parse(list.begin(), list.end()).root().materialize() == expected);
|
||||
|
||||
// iterator pairs: pointers are borrowed, and so are contiguous library
|
||||
// iterators where the input adapter detects them (C++20)
|
||||
CHECK(!json_document::parse(text.data(), text.data() + text.size()).owns_source());
|
||||
const bool contiguous = nlohmann::detail::iterator_input_adapter<std::vector<char>::const_iterator>::supports_bulk_scan;
|
||||
const json_document from_iterators = json_document::parse(chars.cbegin(), chars.cend());
|
||||
CHECK(from_iterators.owns_source() != contiguous);
|
||||
CHECK((from_iterators.source().data() == chars.data()) == contiguous);
|
||||
CHECK(from_iterators.root().materialize() == expected);
|
||||
const std::string padded = "x" + text + "x";
|
||||
CHECK(json_document::parse(padded.begin() + 1, padded.end() - 1).root().materialize() == expected);
|
||||
CHECK(json_document::parse(chars.cbegin(), chars.cbegin(), false).is_discarded());
|
||||
const std::wstring wide = L"[\"\u00e4\u20ac\", 1]";
|
||||
CHECK(json_document::parse(wide).root().materialize() == json::parse(wide));
|
||||
CHECK(json_document::parse(static_cast<const char*>(nullptr), false).is_discarded());
|
||||
CHECK(json_document::parse("", false).is_discarded());
|
||||
}
|
||||
|
||||
SECTION("document lifetime and reuse")
|
||||
{
|
||||
json_document d;
|
||||
CHECK(d.is_discarded());
|
||||
CHECK(d.root().is_discarded());
|
||||
CHECK(d.node_count() == 0);
|
||||
CHECK(d.memory_usage() == 0);
|
||||
CHECK(d.source().empty());
|
||||
|
||||
const std::string a = "[1,2,3]";
|
||||
const std::string b = "{\"x\":[true]}";
|
||||
d.read(a);
|
||||
CHECK(d.node_count() == 4);
|
||||
CHECK(d.root().materialize() == json::parse(a));
|
||||
d.read(b);
|
||||
CHECK(d.node_count() == 4);
|
||||
CHECK(d.root().materialize() == json::parse(b));
|
||||
d.read("[", false);
|
||||
CHECK(d.is_discarded());
|
||||
|
||||
// views stay valid when the document moves
|
||||
json_document first = json_document::parse(a);
|
||||
const json_view root = first.root();
|
||||
const json_document second = std::move(first);
|
||||
CHECK(root.materialize() == json::parse(a));
|
||||
CHECK(second.root().materialize() == json::parse(a));
|
||||
}
|
||||
|
||||
SECTION("memory")
|
||||
{
|
||||
std::string big = "[";
|
||||
for (int i = 0; i < 10000; ++i)
|
||||
{
|
||||
big += (i != 0 ? ",\"" : "\"") + std::to_string(i) + "\"";
|
||||
}
|
||||
big += ']';
|
||||
json_document d = json_document::parse(big);
|
||||
CHECK(d.node_count() == 10001);
|
||||
const std::size_t before = d.memory_usage();
|
||||
d.shrink_to_fit(); // (invalidates views, like std::vector::shrink_to_fit)
|
||||
CHECK(d.memory_usage() <= before);
|
||||
CHECK(d.node_count() == 10001);
|
||||
CHECK(d.root().materialize() == json::parse(big));
|
||||
CHECK(d.root().size() == 10000);
|
||||
d.shrink_to_fit(); // nothing left to release
|
||||
|
||||
// after reading a smaller text, both the index and the decoded strings
|
||||
// shrink, and the strings are found in their new place
|
||||
std::string escaped = "[";
|
||||
for (int i = 0; i < 1000; ++i)
|
||||
{
|
||||
escaped += (i != 0 ? ",\"a\\n" : "\"a\\n") + std::to_string(i) + "\"";
|
||||
}
|
||||
escaped += ']';
|
||||
json_document reused = json_document::parse(escaped);
|
||||
const std::string smaller = "[\"x\\ty\", [true, \"\\u00e4\"]]"; // NOLINT(modernize-raw-string-literal)
|
||||
reused.read(smaller);
|
||||
const std::size_t grown = reused.memory_usage();
|
||||
reused.shrink_to_fit();
|
||||
CHECK(reused.memory_usage() < grown);
|
||||
CHECK(reused.root().materialize() == json::parse(smaller));
|
||||
|
||||
// an empty document has nothing to release
|
||||
json_document empty;
|
||||
empty.shrink_to_fit();
|
||||
CHECK(empty.memory_usage() == 0);
|
||||
|
||||
// a small document stays in the storage block of the header
|
||||
json_document small = json_document::parse("[1,[2,3],{\"a\":\"b\\n\"}]"); // NOLINT(modernize-raw-string-literal)
|
||||
small.shrink_to_fit();
|
||||
CHECK(small.root().materialize() == json::parse("[1,[2,3],{\"a\":\"b\\n\"}]"));
|
||||
}
|
||||
|
||||
SECTION("source offsets")
|
||||
{
|
||||
const std::string text = R"( {"key": "value", "escaped": "a\nb", "n": 42})";
|
||||
const json_document d = json_document::parse(text);
|
||||
CHECK(d.root().source_offset() == 2);
|
||||
// (element access comes with a later change; the offsets of the
|
||||
// string nodes are checked through materialize() above)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,403 @@
|
||||
// __ _____ _____ _____
|
||||
// __| | __| | | | JSON for Modern C++ (supporting code)
|
||||
// | | |__ | | | | | | version 3.12.0
|
||||
// |_____|_____|_____|_|___| https://github.com/nlohmann/json
|
||||
//
|
||||
// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
#include "doctest_compatibility.h"
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
#if JSON_TEST_USING_MULTIPLE_HEADERS
|
||||
#include <nlohmann/detail/view/builder.hpp>
|
||||
#include <nlohmann/detail/view/string_ref.hpp>
|
||||
#else
|
||||
#include <nlohmann/json_view.hpp> // the single header contains the internal headers
|
||||
#endif
|
||||
using nlohmann::json;
|
||||
|
||||
#include <cstdint>
|
||||
#include <fstream>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <random>
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include <test_data.hpp>
|
||||
|
||||
namespace
|
||||
{
|
||||
using nlohmann::detail::view::document_data;
|
||||
using nlohmann::detail::view::node;
|
||||
|
||||
// the node index of a text; a vector input has no terminating NUL, so that
|
||||
// AddressSanitizer catches any read past the last byte
|
||||
struct built
|
||||
{
|
||||
std::unique_ptr<document_data, document_data::deleter> data{}; // NOLINT(readability-redundant-member-init)
|
||||
std::vector<char> copy{}; // NOLINT(readability-redundant-member-init)
|
||||
bool ok = false;
|
||||
nlohmann::detail::view::parse_failure failure{};
|
||||
};
|
||||
|
||||
template<typename FloatType = double>
|
||||
built build(const std::string& text, bool comments, bool trailing_commas, bool sentinel)
|
||||
{
|
||||
built r;
|
||||
r.data.reset(document_data::create(nlohmann::detail::view::estimate_nodes(text.data(), text.size())));
|
||||
const char* src = text.c_str();
|
||||
if (!sentinel)
|
||||
{
|
||||
r.copy.assign(text.begin(), text.end());
|
||||
src = r.copy.data();
|
||||
}
|
||||
r.ok = nlohmann::detail::view::build < FloatType, !nlohmann::detail::abi_config::strict_nul_handling > (*r.data, src, text.size(), comments, trailing_commas, sentinel, r.failure);
|
||||
r.data->src = src;
|
||||
r.data->base[0] = src;
|
||||
r.data->base[1] = r.data->arena.data();
|
||||
return r;
|
||||
}
|
||||
|
||||
// the value of a subtree, as json::parse would build it
|
||||
json value_of(const document_data& d, const node*& n)
|
||||
{
|
||||
const node& x = *n;
|
||||
++n;
|
||||
switch (static_cast<json::value_t>(x.kind))
|
||||
{
|
||||
case json::value_t::object:
|
||||
{
|
||||
json o = json::object();
|
||||
const node* const end = &x + x.next;
|
||||
while (n != end)
|
||||
{
|
||||
const std::string key(d.str(*n), n->len);
|
||||
++n;
|
||||
o[key] = value_of(d, n);
|
||||
}
|
||||
return o;
|
||||
}
|
||||
case json::value_t::array:
|
||||
{
|
||||
json a = json::array();
|
||||
const node* const end = &x + x.next;
|
||||
while (n != end)
|
||||
{
|
||||
a.push_back(value_of(d, n));
|
||||
}
|
||||
return a;
|
||||
}
|
||||
case json::value_t::string:
|
||||
return std::string(d.str(x), x.len);
|
||||
case json::value_t::boolean:
|
||||
return (x.flags & nlohmann::detail::view::node_flags::is_true) != 0;
|
||||
case json::value_t::number_integer:
|
||||
return static_cast<std::int64_t>(nlohmann::detail::view::integer_bits(x));
|
||||
case json::value_t::number_unsigned:
|
||||
return nlohmann::detail::view::integer_bits(x);
|
||||
case json::value_t::number_float:
|
||||
return json::parse(std::string(d.src + x.off, x.len)).get<double>();
|
||||
case json::value_t::null:
|
||||
case json::value_t::binary:
|
||||
case json::value_t::discarded:
|
||||
default:
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
json value_of(const built& b)
|
||||
{
|
||||
const node* n = b.data->tape;
|
||||
json v = value_of(*b.data, n);
|
||||
CHECK(n == b.data->tape + b.data->tape_size);
|
||||
return v;
|
||||
}
|
||||
|
||||
// accept/reject and the value must match json::parse, for all options and
|
||||
// with and without a NUL after the text
|
||||
void check_same(const std::string& text)
|
||||
{
|
||||
CAPTURE(text)
|
||||
for (int options = 0; options < 4; ++options)
|
||||
{
|
||||
const bool comments = (options & 1) != 0;
|
||||
const bool trailing_commas = (options & 2) != 0;
|
||||
const bool accepted = json::accept(text, comments, trailing_commas);
|
||||
for (const bool sentinel :
|
||||
{
|
||||
true, false
|
||||
})
|
||||
{
|
||||
const built b = build(text, comments, trailing_commas, sentinel);
|
||||
CHECK(b.ok == accepted);
|
||||
if (b.ok && accepted)
|
||||
{
|
||||
CHECK(value_of(b) == json::parse(text, nullptr, true, comments, trailing_commas));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// a small deterministic generator of documents
|
||||
struct generator
|
||||
{
|
||||
std::mt19937 rng{5295}; // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed)
|
||||
|
||||
int r(int n)
|
||||
{
|
||||
return static_cast<int>(rng() % static_cast<unsigned>(n));
|
||||
}
|
||||
|
||||
void ws(std::string& o)
|
||||
{
|
||||
for (int n = r(4) == 0 ? r(12) : r(2); n > 0; --n)
|
||||
{
|
||||
o += " \n\t\r "[r(6)];
|
||||
}
|
||||
}
|
||||
|
||||
void str(std::string& o)
|
||||
{
|
||||
static const char* const pieces[] = {"a", "Z", " ", "~", "\\n", "\\\"", "\\\\", "\\/", "\\u00e9", "\\ud83d\\ude00", "\xc3\xa9", "\xe3\x81\x82", "\xf0\x9f\x98\x80", "\x7f", "\\u001f", "long enough text to leave the first 16 bytes"}; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays)
|
||||
o += '"';
|
||||
for (int n = r(3) == 0 ? r(20) : r(6); n > 0; --n)
|
||||
{
|
||||
o += pieces[r(16)];
|
||||
}
|
||||
o += '"';
|
||||
}
|
||||
|
||||
void num(std::string& o)
|
||||
{
|
||||
static const char* const numbers[] = {"0", "-0", "1", "-1", "12", "123456789", "1234567890123456789", "9223372036854775807", "-9223372036854775808", // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays)
|
||||
"9223372036854775808", "18446744073709551615", "18446744073709551616", "-9223372036854775809",
|
||||
"1.5", "-2.25e-3", "1e10", "1E+2", "0.000001", "3.141592653589793238462643", "1e308", "-1e-400", "123.456e7"
|
||||
};
|
||||
o += numbers[r(22)];
|
||||
}
|
||||
|
||||
void value(std::string& o, int depth)
|
||||
{
|
||||
ws(o);
|
||||
const int k = depth > 5 ? 2 + r(6) : r(8);
|
||||
if (k == 0 || k == 1)
|
||||
{
|
||||
const bool object = k == 0;
|
||||
o += object ? '{' : '[';
|
||||
for (int i = r(5); i > 0; --i)
|
||||
{
|
||||
ws(o);
|
||||
if (object)
|
||||
{
|
||||
str(o);
|
||||
ws(o);
|
||||
o += ':';
|
||||
}
|
||||
value(o, depth + 1);
|
||||
o += i > 1 ? "," : "";
|
||||
}
|
||||
ws(o);
|
||||
o += object ? '}' : ']';
|
||||
}
|
||||
else if (k < 4)
|
||||
{
|
||||
str(o);
|
||||
}
|
||||
else if (k < 6)
|
||||
{
|
||||
num(o);
|
||||
}
|
||||
else
|
||||
{
|
||||
static const char* const literals[] = {"true", "false", "null"}; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays)
|
||||
o += literals[r(3)];
|
||||
}
|
||||
ws(o);
|
||||
}
|
||||
};
|
||||
} // namespace
|
||||
|
||||
TEST_CASE("json_view string_ref")
|
||||
{
|
||||
// std::string_view in C++17, a stand-in with the same members before
|
||||
using nlohmann::detail::view::string_ref;
|
||||
const std::string text = "abc";
|
||||
const string_ref r(text);
|
||||
CHECK(r.length() == 3);
|
||||
CHECK(std::string(r.begin(), r.end()) == "abc");
|
||||
CHECK(r[1] == 'b');
|
||||
CHECK(r != string_ref("abd"));
|
||||
CHECK_FALSE(r != string_ref("abcd", 3));
|
||||
std::ostringstream o;
|
||||
o << r;
|
||||
CHECK(o.str() == "abc");
|
||||
}
|
||||
|
||||
TEST_CASE("json_view builder")
|
||||
{
|
||||
SECTION("scalars and containers")
|
||||
{
|
||||
for (const char* text :
|
||||
{
|
||||
"null", "true", "false", "0", "-0", "42", "-42", "1.5", "\"\"", "\"abc\"", "[]", "{}", "[1,2,3]", "{\"a\":1,\"b\":[true,null]}", // NOLINT(modernize-raw-string-literal)
|
||||
" [ 1 , 2 ] ", "{\"a\" : {\"b\" : {}}}", "[[[]]]", "\"\\u00e4\\n\\ud83d\\ude00\"", "{\"a\":1,\"a\":2}", "18446744073709551616", // NOLINT(modernize-raw-string-literal)
|
||||
"-9223372036854775809", "123456789012345678901234567890", "1e400", "-1e400", "1.7976931348623157e308"
|
||||
})
|
||||
{
|
||||
check_same(text);
|
||||
}
|
||||
|
||||
// the midpoint between the largest double and 2^1024 rounds to
|
||||
// infinity (an overflow), one less to the largest double: with more
|
||||
// than 19 digits, Eisel-Lemire cannot decide these, and the overflow
|
||||
// check needs the exact comparison with the midpoint
|
||||
const std::string midpoint = "179769313486231580793728971405303415079934132710037826936173778980444968292764750946649017977587207096330286416692887910946555547851940402630657488671505820681908902000708383676273854845817711531764475730270069855571366959622842914819860834936475292719074168444365510704342711559699508093042880177904174497792";
|
||||
const std::string below = "179769313486231580793728971405303415079934132710037826936173778980444968292764750946649017977587207096330286416692887910946555547851940402630657488671505820681908902000708383676273854845817711531764475730270069855571366959622842914819860834936475292719074168444365510704342711559699508093042880177904174497791";
|
||||
check_same(midpoint);
|
||||
check_same("-" + midpoint);
|
||||
check_same(below);
|
||||
check_same("[" + below + "," + midpoint + "]");
|
||||
|
||||
// the check uses the floating-point type of the document: with float,
|
||||
// the view rejects what parse() rejects (out_of_range.406), and a
|
||||
// double document is not affected
|
||||
using float_json = nlohmann::basic_json<std::map, std::vector, std::string, bool, std::int64_t, std::uint64_t, float>;
|
||||
CHECK_FALSE(float_json::accept("1e39"));
|
||||
CHECK(float_json::accept("3.4028235e38"));
|
||||
CHECK_FALSE(float_json::accept("3.4028236e38"));
|
||||
for (const char* text :
|
||||
{
|
||||
"1e39", "-1e39", "3.4028235e38", "-3.4028235e38", "3.4028236e38", "-3.4028236e38", "3.4028234663852886e38", "1e38",
|
||||
"340282356779733661637539395458142568448", "340282356779733661637539395458142568447.99", "0.00034028236e42",
|
||||
"[1.5e38, 3.5e38]", "{\"a\": 1e-50, \"b\": 1e39}"
|
||||
})
|
||||
{
|
||||
CAPTURE(text)
|
||||
const bool float_accepted = float_json::accept(text);
|
||||
for (const bool sentinel :
|
||||
{
|
||||
true, false
|
||||
})
|
||||
{
|
||||
const built f = build<float>(text, false, false, sentinel);
|
||||
CHECK(f.ok == float_accepted);
|
||||
if (!f.ok)
|
||||
{
|
||||
CHECK(f.failure.code == nlohmann::detail::view::error_code::number_overflow);
|
||||
}
|
||||
CHECK(build<double>(text, false, false, sentinel).ok == json::accept(text));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
SECTION("malformed input")
|
||||
{
|
||||
for (const char* text :
|
||||
{
|
||||
"", " ", "[", "]", "{", "}", "[1,]", "{\"a\":1,}", "[1 2]", "{\"a\" 1}", "{1:2}", "tru", "nul", "fals", "truex", "-", "01", "1.", ".5", "1e", "1e+",
|
||||
"\"", "\"abc", "\"\\x\"", "\"\\u12\"", "\"\\u12G4\"", "\"\\ud800\"", "\"\\udc00\"", "\"\\ud800\\u0041\"", "\"\x01\"", "\"\xff\"", "\"\xc3\"", // NOLINT(modernize-raw-string-literal)
|
||||
"\"\xe0\x80\x80\"", "\"\xed\xa0\x80\"", "[1]x", "[1] [2]", "/", "/*", "/* */ 1", "// c\n1", "1 // c", "[1,/*c*/2]", "[1,2,]"
|
||||
})
|
||||
{
|
||||
check_same(text);
|
||||
}
|
||||
}
|
||||
|
||||
SECTION("NUL, BOM, and whitespace")
|
||||
{
|
||||
// a NUL inside a string is a control character, as for json::parse
|
||||
// (where a NUL ends the input, it does so only between values)
|
||||
for (const bool sentinel :
|
||||
{
|
||||
true, false
|
||||
})
|
||||
{
|
||||
const built b = build(std::string("[\"ab\0cd\"]", 9), false, false, sentinel);
|
||||
CHECK(!b.ok);
|
||||
CHECK(b.failure.code == nlohmann::detail::view::error_code::string_control_character);
|
||||
CHECK(b.failure.offset == 4);
|
||||
}
|
||||
check_same(std::string("[1]\0garbage", 11));
|
||||
check_same(std::string("[1\0]", 4));
|
||||
check_same(std::string("[1, // c\0\n2]", 12));
|
||||
check_same(std::string("[1, /* c\0 */ 2]", 15));
|
||||
check_same("\xEF\xBB\xBF[1]");
|
||||
check_same("\xEF\xBB[1]");
|
||||
check_same(" \t\r\n 7 \n");
|
||||
for (const char* text :
|
||||
{"[1]\r", "[1]\n", "[1]\r\n", "[1,\r2]", "[1,\r\n2]", "7\r", "\"x\"\r", "{\"a\":\r\n1}\r", "[\n 1,\n 2\n]", "{\n \"a\": [\n 1\n ]\n}"
|
||||
})
|
||||
{
|
||||
check_same(text);
|
||||
}
|
||||
}
|
||||
|
||||
SECTION("deep nesting")
|
||||
{
|
||||
// the open containers beyond 64 levels live on the heap
|
||||
for (const std::size_t depth :
|
||||
{
|
||||
63u, 64u, 65u, 1000u, 100000u
|
||||
})
|
||||
{
|
||||
const std::string arrays = std::string(depth, '[') + std::string(depth, ']');
|
||||
const built b = build(arrays, false, false, false);
|
||||
REQUIRE(b.ok);
|
||||
CHECK(b.data->tape_size == depth);
|
||||
CHECK(b.data->tape[0].next == depth);
|
||||
std::string objects;
|
||||
for (std::size_t i = 0; i < depth; ++i)
|
||||
{
|
||||
objects += "{\"a\":";
|
||||
}
|
||||
objects += '1' + std::string(depth, '}');
|
||||
const built o = build(objects, false, false, false);
|
||||
REQUIRE(o.ok);
|
||||
CHECK(o.data->tape_size == (2 * depth) + 1);
|
||||
CHECK(!build(std::string(depth, '[') + std::string(depth - 1, ']'), false, false, false).ok);
|
||||
}
|
||||
}
|
||||
|
||||
SECTION("generated documents and damaged copies")
|
||||
{
|
||||
generator g;
|
||||
for (int i = 0; i < 3000; ++i)
|
||||
{
|
||||
std::string text;
|
||||
g.value(text, 0);
|
||||
check_same(text);
|
||||
// damage: flip one byte, or cut the text
|
||||
std::string damaged = text;
|
||||
const auto at = static_cast<std::size_t>(g.r(static_cast<int>(damaged.size())));
|
||||
static const char replacements[] = {'x', '"', '\\', ',', ':', ']', '}', '[', '{', '1', '-', '.', 'e', '\0', '\n', '/'}; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays)
|
||||
damaged[at] = replacements[g.r(16)];
|
||||
check_same(damaged);
|
||||
check_same(text.substr(0, at));
|
||||
}
|
||||
}
|
||||
|
||||
SECTION("test files")
|
||||
{
|
||||
for (const char* name :
|
||||
{
|
||||
"/json.org/1.json", "/json.org/2.json", "/json.org/3.json", "/json.org/4.json", "/json.org/5.json",
|
||||
"/json_testsuite/sample.json", "/nativejson-benchmark/canada.json", "/nativejson-benchmark/citm_catalog.json",
|
||||
"/nativejson-benchmark/twitter.json", "/json_tests/pass1.json", "/json_tests/pass2.json", "/json_tests/pass3.json"
|
||||
})
|
||||
{
|
||||
CAPTURE(name)
|
||||
std::ifstream f(std::string(TEST_DATA_DIRECTORY) + name, std::ios::binary);
|
||||
std::stringstream ss;
|
||||
ss << f.rdbuf();
|
||||
const std::string text = ss.str();
|
||||
REQUIRE(!text.empty());
|
||||
const built b = build(text, false, false, true);
|
||||
REQUIRE(b.ok);
|
||||
CHECK(value_of(b) == json::parse(text));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
// __ _____ _____ _____
|
||||
// __| | __| | | | JSON for Modern C++ (supporting code)
|
||||
// | | |__ | | | | | | version 3.12.0
|
||||
// |_____|_____|_____|_|___| https://github.com/nlohmann/json
|
||||
//
|
||||
// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
#include "doctest_compatibility.h"
|
||||
|
||||
// All other tests keep the library's macros (JSON_TEST_KEEP_MACROS). This one
|
||||
// includes json_view.hpp as users do, so that json.hpp undefines its macros
|
||||
// (JSON_HAS_CPP_17, JSON_STRICT_NUL_HANDLING, ...) before the view is compiled.
|
||||
#undef JSON_TEST_KEEP_MACROS
|
||||
#include <nlohmann/json_view.hpp>
|
||||
|
||||
#include <string>
|
||||
#include <type_traits>
|
||||
|
||||
#if (defined(__cplusplus) && __cplusplus >= 201703L) || (defined(_MSVC_LANG) && _MSVC_LANG >= 201703L)
|
||||
#include <string_view>
|
||||
#define JSON_VIEW_TEST_HAS_STRING_VIEW 1
|
||||
#else
|
||||
#define JSON_VIEW_TEST_HAS_STRING_VIEW 0
|
||||
#endif
|
||||
|
||||
// the view's own macros do not leak
|
||||
#if defined(NLOHMANN_VIEW_LIKELY) || defined(NLOHMANN_VIEW_UNLIKELY) || defined(NLOHMANN_VIEW_ALWAYS_INLINE) || defined(NLOHMANN_VIEW_NOINLINE) \
|
||||
|| defined(NLOHMANN_VIEW_NODISCARD) || defined(NLOHMANN_VIEW_THROW) || defined(NLOHMANN_VIEW_HAS_CPP_17) || defined(NLOHMANN_VIEW_LITTLE_ENDIAN) \
|
||||
|| defined(NLOHMANN_VIEW_REPEAT16)
|
||||
#error "json_view.hpp leaks a macro"
|
||||
#endif
|
||||
|
||||
TEST_CASE("json_view without the library's macros")
|
||||
{
|
||||
// (this file also gets C++17 builds: it mentions JSON_HAS_CPP_17)
|
||||
#if JSON_VIEW_TEST_HAS_STRING_VIEW
|
||||
CHECK(std::is_same<nlohmann::json_view::string_view_t, std::string_view>::value);
|
||||
#endif
|
||||
const std::string text = "[1, 2.5, \"x\"]";
|
||||
const nlohmann::json_document d = nlohmann::json_document::parse(text);
|
||||
CHECK(d.root().size() == 3);
|
||||
CHECK(d.root().materialize() == nlohmann::json::parse(text));
|
||||
// the NUL handling of the library's configuration
|
||||
const std::string with_nul("[1]\0x", 5);
|
||||
CHECK(nlohmann::json_document::accept(with_nul) == nlohmann::json::accept(with_nul));
|
||||
}
|
||||
Reference in new issue
Block a user