amorfo77/json
1
0
Fork 0
mirror of https://github.com/nlohmann/json.git synced 2026-09-11 18:57:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'fix/parser-callback-duplicate-key' into fix/allow-exceptions-408

Browse Source 
Resolve conflicts from the duplicate-key-stash fix landing on the base branch:
- include/nlohmann/detail/input/json_sax.hpp: take the base branch's already-
  merged remove_discarded_value()/resolve_duplicate_key_stash() implementation
  wholesale; this branch's own change (JSON_THROW -> return parse_error(...)
  for excessive object/array size, honoring allow_exceptions=false) applied
  cleanly elsewhere in the same file.
- tests/src/unit-regression2.cpp / unit-regression3.cpp: the 3-way merge
  duplicated the "duplicate key" test case into both files and appended this
  branch's new "excessive binary container size" test to the now-superseded
  unit-regression2.cpp; keep unit-regression2.cpp as already split and append
  only the genuinely new test to the end of unit-regression3.cpp.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>
This commit is contained in:
Niels Lohmann
2026-09-11 10:34:56 +02:00
parent a7f38e7aeb c03f8dfce8
commit 6ffc69c6dc
35 changed files with 5805 additions and 3535 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/CMakeLists.txt
+86
View File
@@ -2,6 +2,9 @@ cmake_minimum_required(VERSION 3.13...4.0)
option(JSON_Valgrind "Execute test suite with Valgrind." OFF)
option(JSON_FastTests "Skip expensive/slow tests." OFF)
option(JSON_TestSimdutf "Build the unit tests against the simdutf UTF-8 validation backend." OFF)
set(JSON_SIMDUTF_VERSION 9.1.0 CACHE STRING "The simdutf version used by JSON_TestSimdutf.")
set(JSON_32bitTest AUTO CACHE STRING "Enable the 32bit unit test (ON/OFF/AUTO/ONLY).")
set(JSON_TestStandards "" CACHE STRING "The list of standards to test explicitly.")
@@ -194,6 +197,71 @@ if(test_force)
endif()
message(STATUS "${msg}")
#############################################################################
# optionally validate UTF-8 with simdutf (JSON_USE_SIMDUTF)
#############################################################################
# The simdutf backend is opt-in and not vendored, so it is fetched here rather
# than being a checked-in dependency. Everything below hangs off test_main,
# whose usage requirements every test target inherits; the library target and
# the installed CMake package are deliberately left untouched.
if (JSON_TestSimdutf)
# simdutf requires C++17, both to compile itself and to be reachable from
# the library, which keeps its scalar validator below that. Find a tested
# standard that satisfies it.
set(simdutf_standard "")
foreach(cxx_standard ${test_cxx_standards})
if(NOT cxx_standard LESS 17 AND compiler_supports_cpp_${cxx_standard})
set(simdutf_standard ${cxx_standard})
break()
endif()
endforeach()
if("${simdutf_standard}" STREQUAL "")
# Building simdutf would fail outright without a C++17 compiler, and
# even with one it would go unused if no C++17-or-later standard is
# tested. Say so and fall back to the scalar validator rather than
# failing the build.
if(NOT compiler_supports_cpp_17)
set(simdutf_reason "the compiler does not support C++17")
else()
set(simdutf_reason "no tested standard is C++17 or later (testing ${msg_standards})")
endif()
message(WARNING
"JSON_TestSimdutf is enabled, but ${simdutf_reason}. simdutf requires C++17, so it "
"is not fetched and JSON_USE_SIMDUTF is not defined: the tests run against the "
"built-in scalar UTF-8 validator instead. Set JSON_TestStandards to include 17 or "
"later, or build with a compiler that supports C++17.")
else()
if (CMAKE_VERSION VERSION_LESS 3.18)
message(FATAL_ERROR "JSON_TestSimdutf requires CMake 3.18 or later (simdutf's minimum).")
endif()
include(FetchContent)
# simdutf builds its tests and tools by default, and its tests pull
# further dependencies of their own; only the library is needed here
set(SIMDUTF_TESTS OFF CACHE BOOL "" FORCE)
set(SIMDUTF_TOOLS OFF CACHE BOOL "" FORCE)
set(SIMDUTF_BENCHMARKS OFF CACHE BOOL "" FORCE)
set(SIMDUTF_ICONV OFF CACHE BOOL "" FORCE)
FetchContent_Declare(simdutf
URL https://github.com/simdutf/simdutf/archive/refs/tags/v${JSON_SIMDUTF_VERSION}.tar.gz
DOWNLOAD_EXTRACT_TIMESTAMP TRUE
)
FetchContent_MakeAvailable(simdutf)
target_compile_definitions(test_main PUBLIC JSON_USE_SIMDUTF)
target_link_libraries(test_main PUBLIC simdutf::simdutf)
# simdutf.h requires C++17; below that the library keeps its scalar
# validator, so any C++11/14 test targets exercise the fallback and the
# C++17-and-later ones exercise simdutf. Both must agree.
message(STATUS "UTF-8 validation delegated to simdutf ${JSON_SIMDUTF_VERSION} for C++17 and later (JSON_USE_SIMDUTF)")
endif()
endif()
# *DO* use json_test_set_test_options() above this line
json_test_should_build_32bit_test(json_32bit_test json_32bit_test_only "${JSON_32bitTest}")
@@ -230,6 +298,24 @@ json_test_add_test_for(src/unit-comparison.cpp
MAIN test_main CXX_STANDARDS ${test_cxx_standards} ${test_force}
)
# test the parser again with JSON_DIAGNOSTIC_POSITIONS enabled
json_test_set_test_options(test-class_parser_diagnostic_positions
COMPILE_DEFINITIONS JSON_DIAGNOSTIC_POSITIONS=1
)
json_test_add_test_for(src/unit-class_parser.cpp
NAME test-class_parser_diagnostic_positions
MAIN test_main CXX_STANDARDS ${test_cxx_standards} ${test_force}
)
# test diagnostic positions again without regular diagnostics (JSON pointer paths)
json_test_set_test_options(test-diagnostic-positions_only
COMPILE_DEFINITIONS JSON_DIAGNOSTICS=0
)
json_test_add_test_for(src/unit-diagnostic-positions.cpp
NAME test-diagnostic-positions_only
MAIN test_main CXX_STANDARDS ${test_cxx_standards} ${test_force}
)
# *DO NOT* use json_test_set_test_options() below this line
#############################################################################
tests/benchmarks/src/benchmarks.cpp
+38
View File
@@ -81,6 +81,44 @@ BENCHMARK_CAPTURE(ParseString, signed_ints, TEST_DATA_DIRECTORY "/regressi
BENCHMARK_CAPTURE(ParseString, unsigned_ints, TEST_DATA_DIRECTORY "/regression/unsigned_ints.json");
BENCHMARK_CAPTURE(ParseString, small_signed_ints, TEST_DATA_DIRECTORY "/regression/small_signed_ints.json");
//////////////////////////////////////////////////////////////////////////////
// parse pretty-printed JSON from string
//
// Every file in the corpus above is minified or only lightly spaced, so none of
// them exercise the lexer's whitespace handling. Real-world JSON is frequently
// indented - configuration files, pretty-printed API responses, anything kept
// under version control - where insignificant whitespace can outweigh the data.
// Re-serializing a document with an indentation and parsing that keeps the
// content identical to the ParseString row above, so the pair isolates the cost
// of the whitespace alone.
//////////////////////////////////////////////////////////////////////////////
static void ParseIndented(benchmark::State& state, const char* filename, int indent)
{
std::ifstream f(filename);
std::string str((std::istreambuf_iterator<char>(f)), std::istreambuf_iterator<char>());
const std::string indented = json::parse(str).dump(indent);
while (state.KeepRunning())
{
state.PauseTiming();
auto* j = new json();
state.ResumeTiming();
*j = json::parse(indented);
state.PauseTiming();
delete j;
state.ResumeTiming();
}
state.SetBytesProcessed(state.iterations() * indented.size());
}
BENCHMARK_CAPTURE(ParseIndented, jeopardy / 4, TEST_DATA_DIRECTORY "/jeopardy/jeopardy.json", 4);
BENCHMARK_CAPTURE(ParseIndented, canada / 4, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json", 4);
BENCHMARK_CAPTURE(ParseIndented, citm_catalog / 4, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json", 4);
BENCHMARK_CAPTURE(ParseIndented, twitter / 4, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json", 4);
//////////////////////////////////////////////////////////////////////////////
// serialize JSON
//////////////////////////////////////////////////////////////////////////////
tests/src/unit-bjdata.cpp
+65 -3
View File
@@ -2776,6 +2776,53 @@ TEST_CASE("BJData")
CHECK(out_num.at(0) == '{');
CHECK(json::from_bjdata(out_num) == j_num);
}
SECTION("ndarray with out-of-range _ArrayData_ elements stays as object")
{
// each element is cast to the (possibly narrower) C++ type
// named by _ArrayType_ before being written; a value that
// does not fit that type would silently wrap instead of
// being reported, so such an object falls back to a plain
// object encoding that still round-trips (see GitHub issue #5403)
// an unsigned element that does not fit uint8
json const j_uint8 = json({{"_ArrayType_", "uint8"}, {"_ArraySize_", {2}}, {"_ArrayData_", {1, 256}}});
const auto out_uint8 = json::to_bjdata(j_uint8);
CHECK(out_uint8.at(0) == '{');
CHECK(json::from_bjdata(out_uint8) == j_uint8);
// a signed element that does not fit int8
json const j_int8 = json({{"_ArrayType_", "int8"}, {"_ArraySize_", {2}}, {"_ArrayData_", {1, 200}}});
const auto out_int8 = json::to_bjdata(j_int8);
CHECK(out_int8.at(0) == '{');
CHECK(json::from_bjdata(out_int8) == j_int8);
// a negative element is likewise out of range for an
// unsigned _ArrayType_
json const j_uint16_neg = json({{"_ArrayType_", "uint16"}, {"_ArraySize_", {2}}, {"_ArrayData_", {1, -1}}});
const auto out_uint16_neg = json::to_bjdata(j_uint16_neg);
CHECK(out_uint16_neg.at(0) == '{');
CHECK(json::from_bjdata(out_uint16_neg) == j_uint16_neg);
// a double element that overflows to infinity when narrowed
// to the "single" (float) precision named by _ArrayType_
json const j_single = json({{"_ArrayType_", "single"}, {"_ArraySize_", {2}}, {"_ArrayData_", {1.5, 1e40}}});
const auto out_single = json::to_bjdata(j_single);
CHECK(out_single.at(0) == '{');
CHECK(json::from_bjdata(out_single) == j_single);
// in-range boundary values still use the compact ndarray encoding
json const j_uint8_ok = json({{"_ArrayType_", "uint8"}, {"_ArraySize_", {2}}, {"_ArrayData_", {0, 255}}});
CHECK(json::to_bjdata(j_uint8_ok) == std::vector<uint8_t>({'[', '$', 'U', '#', '[', 'i', 2, ']', 0, 255}));
json const j_int8_ok = json({{"_ArrayType_", "int8"}, {"_ArraySize_", {2}}, {"_ArrayData_", {-128, 127}}});
CHECK(json::to_bjdata(j_int8_ok) == std::vector<uint8_t>({'[', '$', 'i', '#', '[', 'i', 2, ']', 0x80, 0x7F}));
json const j_single_ok = json({{"_ArrayType_", "single"}, {"_ArraySize_", {1}}, {"_ArrayData_", {1.5}}});
const auto out_single_ok = json::to_bjdata(j_single_ok);
CHECK(out_single_ok.at(0) == '[');
CHECK(json::from_bjdata(out_single_ok) == json({1.5f}));
}
}
}
@@ -3288,8 +3335,10 @@ TEST_CASE("BJData")
CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vR1), "[json.exception.parse_error.113] parse error at byte 6: syntax error while parsing BJData size: ndarray dimensional vector is not allowed", json::parse_error&);
CHECK(json::from_bjdata(vR1, true, false).is_discarded());
// a dimension vector that opens another one is rejected where the
// nested '[' is read, rather than after it has been descended into
std::vector<uint8_t> const vR2 = {'[', '$', 'i', '#', '[', '#', '[', 'i', 1, ']', ']', 1};
CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vR2), "[json.exception.parse_error.113] parse error at byte 11: syntax error while parsing BJData size: expected length type specification (U, i, u, I, m, l, M, L) after '#'; last byte: 0x5D", json::parse_error&);
CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vR2), "[json.exception.parse_error.113] parse error at byte 7: syntax error while parsing BJData size: ndarray dimensional vector is not allowed", json::parse_error&);
CHECK(json::from_bjdata(vR2, true, false).is_discarded());
std::vector<uint8_t> const vR3 = {'[', '#', '[', 'i', '2', 'i', 2, ']'};
@@ -3297,7 +3346,7 @@ TEST_CASE("BJData")
CHECK(json::from_bjdata(vR3, true, false).is_discarded());
std::vector<uint8_t> const vR4 = {'[', '$', 'i', '#', '[', '$', 'i', '#', '[', 'i', 1, ']', 1};
CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vR4), "[json.exception.parse_error.110] parse error at byte 14: syntax error while parsing BJData number: unexpected end of input", json::parse_error&);
CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vR4), "[json.exception.parse_error.113] parse error at byte 9: syntax error while parsing BJData size: ndarray dimensional vector is not allowed", json::parse_error&);
CHECK(json::from_bjdata(vR4, true, false).is_discarded());
std::vector<uint8_t> const vR5 = {'[', '$', 'i', '#', '[', '[', '[', ']', ']', ']'};
@@ -3305,12 +3354,25 @@ TEST_CASE("BJData")
CHECK(json::from_bjdata(vR5, true, false).is_discarded());
std::vector<uint8_t> const vR6 = {'[', '$', 'i', '#', '[', '$', 'i', '#', '[', 'i', '2', 'i', 2, ']'};
CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vR6), "[json.exception.parse_error.112] parse error at byte 14: syntax error while parsing BJData size: ndarray can not be recursive", json::parse_error&);
CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vR6), "[json.exception.parse_error.113] parse error at byte 9: syntax error while parsing BJData size: ndarray dimensional vector is not allowed", json::parse_error&);
CHECK(json::from_bjdata(vR6, true, false).is_discarded());
std::vector<uint8_t> const vH = {'[', 'H', '[', '#', '[', '$', 'i', '#', '[', 'i', '2', 'i', 2, ']'};
CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vH), "[json.exception.parse_error.113] parse error at byte 3: syntax error while parsing BJData size: ndarray dimensional vector is not allowed", json::parse_error&);
CHECK(json::from_bjdata(vH, true, false).is_discarded());
// Every "#[" of this chain used to open another dimension vector
// and cost several stack frames before anything was rejected, so a
// long enough chain crashed the process (see #5104). The nested
// vector is refused where it is read, so the length is irrelevant.
std::vector<uint8_t> vRdeep = {'['};
for (std::size_t i = 0; i < 100000; ++i)
{
vRdeep.push_back('#');
vRdeep.push_back('[');
}
CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vRdeep), "[json.exception.parse_error.113] parse error at byte 5: syntax error while parsing BJData size: ndarray dimensional vector is not allowed", json::parse_error&);
CHECK(json::from_bjdata(vRdeep, true, false).is_discarded());
}
SECTION("objects")
tests/src/unit-bson.cpp
+85
View File
@@ -1150,6 +1150,91 @@ TEST_CASE("BSON document size mismatch")
}
}
TEST_CASE("BSON nesting does not consume the call stack")
{
// An embedded document or array used to be read by calling back into the
// document reader, so the native call stack grew with the nesting depth of
// the input (#5104). The open documents are kept on a heap stack now.
//
// Deeply nested values must not be compared, copied or dumped here: those
// operations are still recursive and would reintroduce the crash.
// A document nested deeply enough to have crashed. The bytes are built
// here rather than with to_bson(), because the writer still recurses once
// per level and would overflow the stack before the reader is ever
// reached. Every level is
// <int32 size> 0x03 'a' 0x00 <inner document> 0x00
// so a level is eight bytes larger than the one it holds, and the sizes
// can be filled in from the outside in.
const std::size_t depth = 30000;
std::vector<uint8_t> input;
input.reserve(5 + (8 * depth));
for (std::size_t i = 0; i < depth; ++i)
{
const auto size = static_cast<std::uint32_t>(5 + (8 * (depth - i)));
input.push_back(static_cast<uint8_t>(size & 0xFF));
input.push_back(static_cast<uint8_t>((size >> 8) & 0xFF));
input.push_back(static_cast<uint8_t>((size >> 16) & 0xFF));
input.push_back(static_cast<uint8_t>((size >> 24) & 0xFF));
input.push_back(0x03); // embedded document
input.push_back('a');
input.push_back(0x00);
}
// the innermost document is empty, then one terminator closes each level
input.insert(input.end(), {0x05, 0x00, 0x00, 0x00, 0x00});
input.insert(input.end(), depth, 0x00);
SECTION("a well-formed deep document is read through the SAX interface")
{
SaxCountdown accept_all(1000000);
CHECK(json::sax_parse(input, &accept_all, json::input_format_t::bson));
}
SECTION("a well-formed deep document is read into a value")
{
json j = json::from_bson(input);
// walked rather than compared: comparing, copying or dumping a value
// this deep is still recursive
std::size_t measured = 0;
const json* q = &j;
while (q->is_object() && !q->empty())
{
q = &q->begin().value();
++measured;
}
CHECK(measured == depth);
}
SECTION("embedded documents and arrays are still read the same way")
{
const json values = {{"a", {{"b", {{"c", 1}}}}}};
CHECK(json::from_bson(json::to_bson(values)) == values);
const json array = {{"a", {1, 2, 3}}};
CHECK(json::from_bson(json::to_bson(array)) == array);
const json mixed = {{"a", {json{{"x", 1}}, json{{"y", 2}}}}};
CHECK(json::from_bson(json::to_bson(mixed)) == mixed);
CHECK(json::from_bson(json::to_bson(json::object())) == json::object());
}
SECTION("a size that does not match is still reported per document")
{
// the embedded document claims one byte too many
std::vector<uint8_t> const bad =
{
0x15, 0x00, 0x00, 0x00, 0x03, 'a', 0x00,
0x0D, 0x00, 0x00, 0x00, 0x08, 'b', 0x00, 0x01, 0x00,
0x00
};
json _;
CHECK_THROWS_AS(_ = json::from_bson(bad), json::parse_error&);
CHECK(json::from_bson(bad, true, false).is_discarded());
}
}
TEST_CASE("BSON numerical data")
{
SECTION("number")
tests/src/unit-cbor.cpp
+139
View File
@@ -2035,6 +2035,145 @@ TEST_CASE("CBOR definite length equal to the indefinite-length sentinel")
}
}
TEST_CASE("CBOR nesting does not consume the call stack")
{
// Containers used to be read by calling back into the value reader once
// per element, and a tag by calling it for the tagged value, so the native
// call stack grew with the nesting depth of the input. Each of the three
// costs a single byte to encode -- 0x9F, 0x81 and 0xC2 -- so a payload of
// repeated bytes crashed the process (#5104). The containers are kept on a
// heap stack now, and a tag is read in a loop.
//
// Deeply nested values must not be compared, copied or dumped here: those
// operations are still recursive and would reintroduce the crash.
json _;
SECTION("indefinite-length containers")
{
const std::vector<uint8_t> input(500000, 0x9F);
CHECK_THROWS_WITH_AS(_ = json::from_cbor(input), "[json.exception.parse_error.110] parse error at byte 500001: syntax error while parsing CBOR value: unexpected end of input", json::parse_error&);
CHECK(json::from_cbor(input, true, false).is_discarded());
}
SECTION("definite-length containers")
{
const std::vector<uint8_t> input(500000, 0x81);
CHECK_THROWS_WITH_AS(_ = json::from_cbor(input), "[json.exception.parse_error.110] parse error at byte 500001: syntax error while parsing CBOR value: unexpected end of input", json::parse_error&);
CHECK(json::from_cbor(input, true, false).is_discarded());
}
SECTION("tags")
{
// a tag is not a value of its own, so a chain of them used to recurse
const std::vector<uint8_t> input(500000, 0xC2);
CHECK_THROWS_WITH_AS(_ = json::from_cbor(input, true, true, json::cbor_tag_handler_t::ignore), "[json.exception.parse_error.110] parse error at byte 500001: syntax error while parsing CBOR value: unexpected end of input", json::parse_error&);
CHECK(json::from_cbor(input, true, false, json::cbor_tag_handler_t::ignore).is_discarded());
}
SECTION("a well-formed deep value is read through the SAX interface")
{
std::vector<uint8_t> input(200000, 0x9F);
input.insert(input.end(), 200000, 0xFF);
SaxCountdown accept_all(1000000);
CHECK(json::sax_parse(input, &accept_all, json::input_format_t::cbor));
}
SECTION("a well-formed deep value is read into a value")
{
const std::size_t depth = 10000;
std::vector<uint8_t> input(depth, 0x81);
input.push_back(0x00);
json j = json::from_cbor(input);
std::size_t measured = 0;
const json* p = &j;
while (p->is_array() && !p->empty())
{
p = &p->front();
++measured;
}
CHECK(measured == depth);
CHECK(p->is_number());
}
SECTION("containers are still read the same way")
{
CHECK(json::from_cbor(std::vector<uint8_t>({0x80})) == json::array());
CHECK(json::from_cbor(std::vector<uint8_t>({0xA0})) == json::object());
CHECK(json::from_cbor(std::vector<uint8_t>({0x9F, 0xFF})) == json::array());
CHECK(json::from_cbor(std::vector<uint8_t>({0xBF, 0xFF})) == json::object());
CHECK(json::from_cbor(std::vector<uint8_t>({0x9F, 0x01, 0x02, 0xFF})) == json({1, 2}));
CHECK(json::from_cbor(std::vector<uint8_t>({0xBF, 0x61, 'a', 0x01, 0xFF})) == json({{"a", 1}}));
// definite and indefinite forms nested inside each other
CHECK(json::from_cbor(std::vector<uint8_t>({0x9F, 0x82, 0x01, 0x02, 0xA1, 0x61, 'k', 0xBF, 0xFF, 0xFF})) == json({{1, 2}, {{"k", json::object()}}}));
}
SECTION("tagged values are still read the same way")
{
const auto ignore = json::cbor_tag_handler_t::ignore;
CHECK(json::from_cbor(std::vector<uint8_t>({0xC2, 0x01}), true, true, ignore) == json(1));
// a chain of tags resolves to the value that follows it
CHECK(json::from_cbor(std::vector<uint8_t>({0xC2, 0xC2, 0xC2, 0x01}), true, true, ignore) == json(1));
// a tag inside a container, and one in front of a container
CHECK(json::from_cbor(std::vector<uint8_t>({0x82, 0xC2, 0x01, 0x02}), true, true, ignore) == json({1, 2}));
CHECK(json::from_cbor(std::vector<uint8_t>({0xC2, 0x82, 0x01, 0x02}), true, true, ignore) == json({1, 2}));
}
}
TEST_CASE("CBOR indefinite-length strings do not recurse per chunk")
{
// Reading an indefinite-length string or byte array used to call itself
// once per chunk, so a payload of repeated 0x7F (or 0x5F) bytes exhausted
// the call stack before any of the input was rejected. The open levels are
// counted now, and the levels below prove the reader still reads the same
// values and reports the same errors at the same byte offsets.
json _;
SECTION("many open levels are reported, not crashed on")
{
const std::vector<uint8_t> input(200000, 0x7F);
CHECK_THROWS_WITH_AS(_ = json::from_cbor(input), "[json.exception.parse_error.110] parse error at byte 200001: syntax error while parsing CBOR string: unexpected end of input", json::parse_error&);
CHECK(json::from_cbor(input, true, false).is_discarded());
}
SECTION("many open levels are reported, not crashed on (binary)")
{
const std::vector<uint8_t> input(200000, 0x5F);
CHECK_THROWS_WITH_AS(_ = json::from_cbor(input), "[json.exception.parse_error.110] parse error at byte 200001: syntax error while parsing CBOR binary: unexpected end of input", json::parse_error&);
CHECK(json::from_cbor(input, true, false).is_discarded());
}
SECTION("chunks are still concatenated")
{
CHECK(json::from_cbor(std::vector<uint8_t>({0x7F, 0xFF})) == json(""));
CHECK(json::from_cbor(std::vector<uint8_t>({0x7F, 0x61, 0x61, 0xFF})) == json("a"));
// nested indefinite-length strings are concatenated across levels
CHECK(json::from_cbor(std::vector<uint8_t>({0x7F, 0x7F, 0x61, 0x61, 0xFF, 0x61, 0x62, 0xFF})) == json("ab"));
CHECK(json::from_cbor(std::vector<uint8_t>({0x7F, 0x7F, 0x7F, 0x61, 0x7A, 0xFF, 0xFF, 0xFF})) == json("z"));
CHECK(json::from_cbor(std::vector<uint8_t>({0xA1, 0x7F, 0x61, 0x61, 0xFF, 0x01})) == json({{"a", 1}}));
}
SECTION("chunks are still concatenated (binary)")
{
CHECK(json::from_cbor(std::vector<uint8_t>({0x5F, 0x41, 0x61, 0xFF})) == json::binary({0x61}));
CHECK(json::from_cbor(std::vector<uint8_t>({0x5F, 0x5F, 0x41, 0x61, 0xFF, 0x41, 0x62, 0xFF})) == json::binary({0x61, 0x62}));
}
SECTION("a chunk that is not a string is still rejected")
{
CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector<uint8_t>({0x7F, 0x7F, 0x00})), "[json.exception.parse_error.113] parse error at byte 3: syntax error while parsing CBOR string: expected length specification (0x60-0x7B) or indefinite string type (0x7F); last byte: 0x00", json::parse_error&);
CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector<uint8_t>({0x5F, 0x5F, 0x00})), "[json.exception.parse_error.113] parse error at byte 3: syntax error while parsing CBOR binary: expected length specification (0x40-0x5B) or indefinite binary array type (0x5F); last byte: 0x00", json::parse_error&);
}
SECTION("a break marker outside an indefinite-length string is not a string")
{
// 0xFF only closes a string that was opened; on its own it is not one
CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector<uint8_t>({0xA1, 0xFF, 0x01})), "[json.exception.parse_error.113] parse error at byte 2: syntax error while parsing CBOR string: expected length specification (0x60-0x7B) or indefinite string type (0x7F); last byte: 0xFF", json::parse_error&);
}
}
TEST_CASE("CBOR roundtrips" * doctest::skip())
{
SECTION("input from flynn")
tests/src/unit-class_lexer.cpp
+433
View File
@@ -12,6 +12,11 @@
#include <nlohmann/json.hpp>
using nlohmann::json;
#include <cstdlib> // strtod
#include <sstream> // stringstream
#include <string> // string
#include <vector> // vector
namespace
{
// shortcut to scan a string literal
@@ -224,3 +229,431 @@ TEST_CASE("lexer class")
CHECK((scan_string("/**//**//**/", true) == json::lexer::token_type::end_of_input));
}
}
TEST_CASE("lexer number fast path")
{
// The contiguous fast path (used for pointer/string input) must agree with
// the streaming byte path (used for std::istream) on token type, numeric
// value, and round-trip text for every well-formed number, and reject the
// same malformed numbers with the same message.
SECTION("contiguous vs streaming parity")
{
const std::vector<std::string> numbers =
{
"0", "-0", "1", "-1", "42", "-42", "10", "100", "1234567890",
"0.0", "-0.0", "3.14", "-3.14", "0.5", "-0.001", "123.456789",
"1e0", "1E0", "1e10", "1e-10", "1e+10", "1.5e3", "-2.5E-4",
"9223372036854775807", // INT64_MAX -> unsigned
"9223372036854775808", // INT64_MAX + 1 -> unsigned
"18446744073709551615", // UINT64_MAX -> unsigned
"18446744073709551616", // UINT64_MAX + 1 -> float
"-9223372036854775808", // INT64_MIN -> integer
"-9223372036854775809", // INT64_MIN - 1 -> float
"123456789012345678901234567890", // huge -> float
"0.30000000000000004", "2.2250738585072014e-308", "1e308",
// high-precision / wide-exponent values that exercise the
// std::from_chars (Eisel-Lemire) path beyond the Clinger subset
"1.7976931348623157e308", "1.2345678901234567e-250",
"9007199254740993", "5e-324", "1e-320"
};
for (const auto& n : numbers)
{
const std::string doc = "[" + n + "]";
// contiguous fast path
const json a = json::parse(doc);
// streaming byte path
std::stringstream ss(doc);
const json b = json::parse(ss);
CAPTURE(n);
CHECK(a == b);
CHECK(a.dump() == b.dump());
CHECK(a[0].type() == b[0].type());
}
}
SECTION("significant-digit gate for the Clinger fast path")
{
// Clinger's fast path needs a significand below 2^53, so it cannot
// succeed once the mantissa has 17 or more significant digits (the
// significand would be at least 10^16). The lexer skips the attempt
// there. That is only allowed to save work: every value must still come
// out bit-exactly, and both scanners must agree. In particular the gate
// must not fire for tokens whose leading zeros merely look like extra
// digits - "0.1234567890123456" has 16 significant digits, not 17.
const std::vector<std::string> numbers =
{
"1234567890123456", // 16 significant digits
"12345678901234567", // 17 -> attempt skipped
"123456789012345678", // 18 -> attempt skipped
"0.1234567890123456", // 16: the leading "0" is not significant
"0.12345678901234567", // 17
"0.00000000000000001", // 1, in a long token
"0.000000000000000012345678901234", // 14, in a long token
"-0.0000000000000000000001", // 1, negative
"1.0000000000000000", // 17: trailing zeros are significant here
"10000000000000000", // 17
"9007199254740992", // 2^53
"9007199254740993", // 2^53 + 1
"-65.613616999999977", // canada.json shape
"1.2345678901234567e-250", // 17 with an exponent
"1.234567890123456e-250", // 16 with an exponent
"1e10", "0.0", "-0.0", "0e0", "0.000123"
};
for (const auto& n : numbers)
{
CAPTURE(n);
const std::string doc = "[" + n + "]";
const json a = json::parse(doc); // contiguous fast path
std::stringstream ss(doc);
const json b = json::parse(ss); // streaming byte path
CHECK(a[0].type() == b[0].type());
CHECK(a == b);
if (a[0].is_number_float())
{
const double expected = std::strtod(n.c_str(), nullptr);
CHECK(a[0].get<double>() == expected);
CHECK(b[0].get<double>() == expected);
}
}
}
SECTION("token type classification")
{
CHECK((scan_string("0") == json::lexer::token_type::value_unsigned));
CHECK((scan_string("-1") == json::lexer::token_type::value_integer));
CHECK((scan_string("1.5") == json::lexer::token_type::value_float));
CHECK((scan_string("1e5") == json::lexer::token_type::value_float));
CHECK((scan_string("18446744073709551615") == json::lexer::token_type::value_unsigned));
CHECK((scan_string("18446744073709551616") == json::lexer::token_type::value_float));
CHECK((scan_string("-9223372036854775808") == json::lexer::token_type::value_integer));
CHECK((scan_string("-9223372036854775809") == json::lexer::token_type::value_float));
}
SECTION("malformed numbers are rejected identically")
{
for (const char* bad :
{"-", "1.", "1e", "1e+", "1.2e", "01", "-01", "1..2", "1.2.3"
})
{
CAPTURE(bad);
// the contiguous fast path must decline and let the byte path report
const std::string doc = std::string("[") + bad + "]";
CHECK_FALSE(json::accept(doc));
std::stringstream ss(doc);
CHECK_FALSE(json::accept(ss));
}
}
#if !defined(JSON_NOEXCEPTION)
// these sections parse invalid input, which aborts when exceptions are off
SECTION("exhaustive grammar parity with the streaming path")
{
// The JSON number grammar is encoded twice: once as the scan_number()
// state machine and once as the contiguous fast path. Enumerate every
// short string over the number alphabet and require the two encodings to
// agree exactly - on acceptance, on the reported error, and on the parsed
// value - so they cannot drift apart.
const std::string alphabet = "01.eE+-";
// full outcome of parsing @a doc, so a mismatch in type, value, or error
// message is caught, not just a mismatch in acceptance
const auto outcome = [](const std::string & doc, bool streaming) -> std::string
{
try
{
if (streaming)
{
std::stringstream ss(doc);
const json j = json::parse(ss);
return std::string(j[0].type_name()) + '|' + j.dump();
}
const json j = json::parse(doc);
return std::string(j[0].type_name()) + '|' + j.dump();
}
catch (const json::parse_error& e)
{
return {e.what()};
}
};
std::vector<std::string> mismatches;
std::vector<std::string> tokens{""};
for (std::size_t length = 1; length <= 4; ++length)
{
std::vector<std::string> next;
next.reserve(tokens.size() * alphabet.size());
for (const auto& prefix : tokens)
{
for (const char c : alphabet)
{
next.push_back(prefix + c);
}
}
tokens = next;
for (const auto& token : tokens)
{
const std::string doc = "[" + token + "]";
if (outcome(doc, false) != outcome(doc, true))
{
mismatches.push_back(doc);
}
}
}
// 7 + 49 + 343 + 2401 tokens
CHECK(tokens.size() == 2401);
CAPTURE(mismatches);
CHECK(mismatches.empty());
}
SECTION("error positions match the streaming path")
{
// Rejecting identically is not enough: the fast path must also report the
// error at the same position as the byte path. A number directly followed
// by a newline is the interesting case, because the byte path reaches the
// newline (which resets the column) and then ungets it.
// returns the parse_error message, or "" if the document parsed
const auto contiguous_error = [](const std::string & doc) -> std::string
{
try
{
const json j = json::parse(doc);
static_cast<void>(j);
}
catch (const json::parse_error& e)
{
return {e.what()};
}
return {};
};
const auto streaming_error = [](const std::string & doc) -> std::string
{
try
{
std::stringstream ss(doc);
const json j = json::parse(ss);
static_cast<void>(j);
}
catch (const json::parse_error& e)
{
return {e.what()};
}
return {};
};
for (const char* bad :
{"[01\n]", "[00\n]", "[-01\n]", "{1\n}", "[1\n2]", "[1.2.3\n]",
"[1 \n2]", "[\n1\n2]", "1\n2", "[01\r\n]", "[1e\n]", "[-\n]"
})
{
CAPTURE(bad);
const std::string doc = bad;
const std::string contiguous_what = contiguous_error(doc);
CHECK_FALSE(contiguous_what.empty());
CHECK(contiguous_what == streaming_error(doc));
}
// A number terminated by a newline must report the same position as the
// same number terminated by anything else: scan_number() reads the
// terminator and ungets it, so the reported column is the one reached
// after the number's last character - not the 0 that an unget() across
// the newline used to leave behind.
CHECK(contiguous_error("[01\n]") == contiguous_error("[01 ]"));
CHECK(contiguous_error("[01\n]") ==
"[json.exception.parse_error.101] parse error at line 1, column 3: "
"syntax error while parsing array - unexpected number literal; expected ']'");
// the same for a multi-character token, where the column of the last
// character (the '3' of "-2.5e3") differs from the column it starts at
CHECK(contiguous_error("null -2.5e3\nfalse") == contiguous_error("null -2.5e3 false"));
CHECK(contiguous_error("null -2.5e3\nfalse") ==
"[json.exception.parse_error.101] parse error at line 1, column 11: "
"syntax error while parsing value - unexpected number literal; expected end of input");
}
#endif
}
TEST_CASE("lexer string fast path")
{
// Build a byte string from explicit values: a hex escape in a string
// literal swallows every following hex digit, which makes sequences like
// "\xC3\xA9b" mean something other than they look like.
const auto bytes = [](std::initializer_list<int> values)
{
std::string result;
for (const int value : values)
{
result.push_back(static_cast<char>(value));
}
return result;
};
#if !defined(JSON_NOEXCEPTION)
// the full outcome of parsing @a doc: the parsed value, or the exact error
// message, so a mismatch in either is caught. Only usable with exceptions
// on: parsing invalid input aborts when they are off.
const auto outcome = [](const std::string & doc, bool streaming) -> std::string
{
try
{
if (streaming)
{
std::stringstream ss(doc);
const json j = json::parse(ss);
return j.dump();
}
const json j = json::parse(doc);
return j.dump();
}
// not just parse_error: if a bulk scanner ever let ill-formed UTF-8
// through, dump() would throw type_error.316, and that has to surface
// as a reported mismatch rather than as an uncaught exception
catch (const json::exception& e)
{
return {e.what()};
}
};
#endif
// once at the start of the string, once past the first 8-byte SWAR word, so
// the bulk scanner sees each case with and without a run behind it
const std::vector<std::size_t> offsets{0, 9};
#if !defined(JSON_NOEXCEPTION)
SECTION("exhaustive contiguous vs streaming parity")
{
// ordinary ASCII, both specials, a control byte, characters that make
// the preceding backslash a valid escape, a UTF-8 lead byte of each
// length, a continuation byte, and a byte that is never valid
const std::vector<std::string> alphabet =
{
"a", "\"", "\\", "n", "u", "0", bytes({0x01}),
bytes({0xC3}), bytes({0xA9}), bytes({0xE4}), bytes({0xF0}),
bytes({0x80}), bytes({0xFF})
};
std::vector<std::string> mismatches;
std::vector<std::string> tokens{""};
for (std::size_t length = 1; length <= 3; ++length)
{
std::vector<std::string> next;
next.reserve(tokens.size() * alphabet.size());
for (const auto& prefix : tokens)
{
for (const auto& symbol : alphabet)
{
next.push_back(prefix + symbol);
}
}
tokens = next;
for (const auto& token : tokens)
{
for (const std::size_t offset : offsets)
{
const std::string doc = "[\"" + std::string(offset, 'a') + token + "\"]";
if (outcome(doc, false) != outcome(doc, true))
{
mismatches.push_back(doc);
}
}
}
}
// 13 + 169 + 2197 tokens, each at two offsets
CHECK(tokens.size() == 2197);
CAPTURE(mismatches);
CHECK(mismatches.empty());
}
SECTION("special bytes at every offset of the SWAR stride")
{
// The bulk scanner consumes 8 bytes at a time and then a tail; place
// every kind of byte that ends a run at each offset across two words,
// so multibyte sequences also straddle the word boundary.
const std::vector<std::string> specials =
{
"\"", "\\", bytes({0x01}), bytes({0x1F}), bytes({0x7F}),
bytes({0xC3, 0xA9}), bytes({0xE4, 0xB8, 0xAD}), bytes({0xF0, 0x9F, 0x98, 0x80}),
bytes({0xFF}), bytes({0xC3}), bytes({0xE4, 0xB8})
};
std::vector<std::string> mismatches;
for (std::size_t offset = 0; offset <= 17; ++offset)
{
for (const auto& special : specials)
{
const std::string doc = "[\"" + std::string(offset, 'a') + special + "\"]";
if (outcome(doc, false) != outcome(doc, true))
{
mismatches.push_back(doc);
}
}
}
CAPTURE(mismatches);
CHECK(mismatches.empty());
}
#endif
// json::accept() never throws, so the ranges stay covered without exceptions
SECTION("UTF-8 ranges are accepted and rejected as documented")
{
// The bulk validator must accept exactly what the byte-at-a-time
// scanner accepts, so pin the boundaries of every range it recognizes.
// aggregate, only ever brace-initialized below; default member
// initializers would stop it being an aggregate in C++11
struct utf8_case // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
{
std::string sequence;
bool valid;
const char* description;
};
const std::vector<utf8_case> cases =
{
{bytes({0xC2, 0x80}), true, "U+0080, shortest two-byte"},
{bytes({0xDF, 0xBF}), true, "U+07FF, longest two-byte"},
{bytes({0xC1, 0xBF}), false, "overlong two-byte"},
{bytes({0xC2, 0x7F}), false, "two-byte with bad continuation"},
{bytes({0xE0, 0xA0, 0x80}), true, "U+0800, shortest three-byte"},
{bytes({0xE0, 0x9F, 0xBF}), false, "overlong three-byte"},
{bytes({0xED, 0x9F, 0xBF}), true, "U+D7FF, just below the surrogates"},
{bytes({0xED, 0xA0, 0x80}), false, "surrogate U+D800"},
{bytes({0xED, 0xBF, 0xBF}), false, "surrogate U+DFFF"},
{bytes({0xEE, 0x80, 0x80}), true, "U+E000, just above the surrogates"},
{bytes({0xEF, 0xBF, 0xBF}), true, "U+FFFF"},
{bytes({0xF0, 0x90, 0x80, 0x80}), true, "U+10000, shortest four-byte"},
{bytes({0xF0, 0x8F, 0xBF, 0xBF}), false, "overlong four-byte"},
{bytes({0xF4, 0x8F, 0xBF, 0xBF}), true, "U+10FFFF, highest code point"},
{bytes({0xF4, 0x90, 0x80, 0x80}), false, "above U+10FFFF"},
{bytes({0xF5, 0x80, 0x80, 0x80}), false, "lead byte out of range"},
{bytes({0x80}), false, "bare continuation byte"},
{bytes({0xFF}), false, "byte that never appears in UTF-8"},
{bytes({0xC3}), false, "truncated two-byte"},
{bytes({0xE4, 0xB8}), false, "truncated three-byte"},
{bytes({0xF0, 0x9F, 0x98}), false, "truncated four-byte"}
};
for (const auto& test_case : cases)
{
CAPTURE(test_case.description);
for (const std::size_t offset : offsets)
{
CAPTURE(offset);
const std::string doc = "[\"" + std::string(offset, 'a') + test_case.sequence + "\"]";
CHECK(json::accept(doc) == test_case.valid);
#if !defined(JSON_NOEXCEPTION)
CHECK(outcome(doc, false) == outcome(doc, true));
#endif
}
}
}
}
tests/src/unit-class_parser.cpp
+318
View File
@@ -346,6 +346,50 @@ void trailing_comma_helper(const std::string& s)
}
}
#if JSON_DIAGNOSTIC_POSITIONS
/**
* Validates that the generated JSON object is the same as expected
* Validates that the start position and end position match the start and end of the string
*
* This check assumes that there is no whitespace around the json object in the original string.
*/
void validate_generated_json_and_start_end_pos_helper(const std::string& original_string, const json& j, const json& check)
{
CHECK(j == check);
CHECK(j.start_pos() == 0);
CHECK(j.end_pos() == original_string.size());
}
/**
* Parses the root object from the given root string and validates that the start and end positions for the nested object are correct.
*
* This checks that whitespace around the nested object is included in the start and end positions of the root object.
*/
void validate_start_end_pos_for_nested_obj_helper(const std::string& nested_type_json_str, const std::string& root_type_json_str, const json& expected_json, const json::parser_callback_t& cb = nullptr)
{
json j;
// 1. If callback is provided, use callback version of parse()
if (cb)
{
j = json::parse(root_type_json_str, cb);
}
else
{
j = json::parse(root_type_json_str);
}
// 2. Check if the generated JSON is as expected
// Assumptions: The root_type_json_str does not have any whitespace around the json object
validate_generated_json_and_start_end_pos_helper(root_type_json_str, j, expected_json);
// 3. Get the nested object
const auto& nested = j["nested"];
// 4. Check if the start and end positions are generated correctly for nested objects and arrays
CHECK(nested_type_json_str == root_type_json_str.substr(nested.start_pos(), nested.end_pos() - nested.start_pos()));
}
#endif
} // namespace
TEST_CASE("parser class")
@@ -1724,6 +1768,58 @@ TEST_CASE("parser class")
CHECK (j_filtered2 == json({{"foo", {1, 2}}}));
}
SECTION("filter many members of one container")
{
// Rejecting a value makes the parser remove the placeholder its key
// event stored. Locating that placeholder used to be a scan of the
// whole parent, which made filtering a large container quadratic:
// 128k members took ~25 s. These cases keep many members alive
// while discarding many others, so the removal cost is the whole
// point; they run in milliseconds when the placeholder is erased
// directly.
constexpr int count = 20000;
std::string s = "{";
for (int i = 0; i < count; ++i)
{
// "a<i>" is kept, "z<i>" is discarded
s += "\"a" + std::to_string(i) + "\":" + std::to_string(i) + ",";
s += "\"z" + std::to_string(i) + "\":-1,";
}
s.back() = '}';
const json j_values = json::parse(s, [](int /*unused*/, json::parse_event_t e, const json & parsed) noexcept
{
return !(e == json::parse_event_t::value && parsed == json(-1));
});
CHECK(j_values.size() == count);
CHECK(j_values.at("a0") == json(0));
CHECK(j_values.at("a" + std::to_string(count - 1)) == json(count - 1));
CHECK_FALSE(j_values.contains("z0"));
CHECK_FALSE(j_values.contains("z" + std::to_string(count - 1)));
// the same, but discarding whole containers rather than values,
// which takes the end_object()/end_array() removal path
std::string s_nested = "{";
for (int i = 0; i < count; ++i)
{
s_nested += "\"a" + std::to_string(i) + "\":" + std::to_string(i) + ",";
s_nested += "\"z" + std::to_string(i) + "\":[1,2],";
}
s_nested.back() = '}';
const json j_arrays = json::parse(s_nested, [](int /*unused*/, json::parse_event_t e, const json& /*unused*/) noexcept
{
return e != json::parse_event_t::array_end;
});
CHECK(j_arrays.size() == count);
CHECK(j_arrays.at("a0") == json(0));
CHECK_FALSE(j_arrays.contains("z0"));
CHECK_FALSE(j_arrays.contains("z" + std::to_string(count - 1)));
}
SECTION("filter specific events")
{
SECTION("first closing event")
@@ -1939,6 +2035,228 @@ TEST_CASE("parser class")
CHECK_THROWS_WITH_AS(_ = json::parse("/a", nullptr, true, true), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid comment; expecting '/' or '*' after '/'; last read: '/a'", json::parse_error);
CHECK_THROWS_WITH_AS(_ = json::parse("/*", nullptr, true, true), "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - invalid comment; missing closing '*/'; last read: '/*<U+0000>'", json::parse_error);
}
#if JSON_DIAGNOSTIC_POSITIONS
// Macro for all test cases for start_pos and end_pos
#define SETUP_TESTCASES() \
SECTION("with callback") \
{ \
SECTION("filter nothing") \
{ \
json::parser_callback_t const cb = [](int /*unused*/, json::parse_event_t /*unused*/, json& /*unused*/) noexcept \
{ \
return true; \
}; \
validate_start_end_pos_for_nested_obj_helper(nested_type_json_str, root_type_json_str, expected, cb); \
} \
SECTION("filter element") \
{ \
json::parser_callback_t const cb = [](int /*unused*/, json::parse_event_t event, json& j) noexcept \
{ \
return (event != json::parse_event_t::key && event != json::parse_event_t::value) || j != json("a"); \
}; \
validate_start_end_pos_for_nested_obj_helper(nested_type_json_str, root_type_json_str, filteredExpected, cb); \
} \
} \
SECTION("without callback") \
{ \
validate_start_end_pos_for_nested_obj_helper(nested_type_json_str, root_type_json_str, expected); \
}
SECTION("retrieve start position and end position")
{
SECTION("for object")
{
// Create an object with spaces to test the start and end positions. Spaces will not be included in the
// JSON object, however, the start and end positions should include the spaces from the input JSON string.
const std::string nested_type_json_str = R"({ "a": 1,"b" : "test1"})";
const std::string root_type_json_str = R"({ "nested": )" + nested_type_json_str + R"(, "anotherValue": "test2"})";
auto expected = json({{"nested", {{"a", 1}, {"b", "test1"}}}, {"anotherValue", "test2"}});
auto filteredExpected = expected;
filteredExpected["nested"].erase("a");
SETUP_TESTCASES()
}
SECTION("for array")
{
const std::string nested_type_json_str = R"(["a", "test", 45])";
const std::string root_type_json_str = R"({ "nested": )" + nested_type_json_str + R"(, "anotherValue": "test" })";
auto expected = json({{"nested", {"a", "test", 45}}, {"anotherValue", "test"}});
auto filteredExpected = expected;
filteredExpected["nested"] = json({"test", 45});
SETUP_TESTCASES()
}
SECTION("for array with objects")
{
const std::string nested_type_json_str = R"([{"a": 1, "b": "test"}, {"c": 2, "d": "test2"}])";
const std::string root_type_json_str = R"({ "nested": )" + nested_type_json_str + R"(, "anotherValue": "test" })";
auto expected = json({{"nested", {{{"a", 1}, {"b", "test"}}, {{"c", 2}, {"d", "test2"}}}}, {"anotherValue", "test"}});
auto filteredExpected = expected;
filteredExpected["nested"][0].erase("a");
SETUP_TESTCASES()
auto j = json::parse(root_type_json_str);
auto nested_array = j["nested"];
const auto& nested_obj = nested_array[0];
CHECK(nested_type_json_str.substr(1, 21) == root_type_json_str.substr(nested_obj.start_pos(), nested_obj.end_pos() - nested_obj.start_pos()));
CHECK(nested_type_json_str.substr(24, 22) == root_type_json_str.substr(nested_array[1].start_pos(), nested_array[1].end_pos() - nested_array[1].start_pos()));
}
SECTION("for two levels of nesting objects")
{
const std::string nested_type_json_str = R"({"nested2": {"b": "test"}})";
const std::string root_type_json_str = R"({ "a": 2, "nested": )" + nested_type_json_str + R"(, "anotherValue": "test" })";
auto expected = json({{"a", 2}, {"nested", {{"nested2", {{"b", "test"}}}}}, {"anotherValue", "test"}});
auto filteredExpected = expected;
filteredExpected.erase("a");
SETUP_TESTCASES()
auto j = json::parse(root_type_json_str);
auto nested_obj = j["nested"]["nested2"];
CHECK(nested_type_json_str.substr(12, 13) == root_type_json_str.substr(nested_obj.start_pos(), nested_obj.end_pos() - nested_obj.start_pos()));
}
SECTION("for simple types")
{
SECTION("no nested")
{
SECTION("with callback")
{
json::parser_callback_t const cb = [](int /*unused*/, json::parse_event_t /*unused*/, json& /*unused*/) noexcept
{
return true;
};
// 1. string type
std::string json_str = R"("test")";
auto j = json::parse(json_str, cb);
validate_generated_json_and_start_end_pos_helper(json_str, j, "test");
// 2. number type
json_str = R"(1)";
j = json::parse(json_str, cb);
validate_generated_json_and_start_end_pos_helper(json_str, j, 1);
// 3. boolean type
json_str = R"(true)";
j = json::parse(json_str, cb);
validate_generated_json_and_start_end_pos_helper(json_str, j, true);
// 4. null type
json_str = R"(null)";
j = json::parse(json_str, cb);
validate_generated_json_and_start_end_pos_helper(json_str, j, nullptr);
}
SECTION("without callback")
{
// 1. string type
std::string json_str = R"("test")";
auto j = json::parse(json_str);
validate_generated_json_and_start_end_pos_helper(json_str, j, "test");
// 2. number type
json_str = R"(1)";
j = json::parse(json_str);
validate_generated_json_and_start_end_pos_helper(json_str, j, 1);
json_str = R"(1.001239923)";
j = json::parse(json_str);
validate_generated_json_and_start_end_pos_helper(json_str, j, 1.001239923);
json_str = R"(1.123812389000000)";
j = json::parse(json_str);
validate_generated_json_and_start_end_pos_helper(json_str, j, 1.123812389);
// 3. boolean type
json_str = R"(true)";
j = json::parse(json_str);
validate_generated_json_and_start_end_pos_helper(json_str, j, true);
json_str = R"(false)";
j = json::parse(json_str);
validate_generated_json_and_start_end_pos_helper(json_str, j, false);
// 4. null type
json_str = R"(null)";
j = json::parse(json_str);
validate_generated_json_and_start_end_pos_helper(json_str, j, nullptr);
}
}
SECTION("string type")
{
const std::string nested_type_json_str = R"("test")";
const std::string root_type_json_str = R"({ "a": 1, "nested": )" + nested_type_json_str + R"(, "anotherValue": "test" })";
auto expected = json({{"nested", "test"}, {"anotherValue", "test"}, {"a", 1}});
auto filteredExpected = expected;
filteredExpected.erase("a");
SETUP_TESTCASES()
}
SECTION("number type")
{
const std::string nested_type_json_str = R"(2)";
const std::string root_type_json_str = R"({ "a": 1, "nested": )" + nested_type_json_str + R"(, "anotherValue": "test" })";
auto expected = json({{"nested", 2}, {"anotherValue", "test"}, {"a", 1}});
auto filteredExpected = expected;
filteredExpected.erase("a");
SETUP_TESTCASES()
}
SECTION("boolean type")
{
const std::string nested_type_json_str = R"(true)";
const std::string root_type_json_str = R"({ "a": 1, "nested": )" + nested_type_json_str + R"(, "anotherValue": "test" })";
auto expected = json({{"nested", true}, {"anotherValue", "test"}, {"a", 1}});
auto filteredExpected = expected;
filteredExpected.erase("a");
SETUP_TESTCASES()
}
SECTION("null type")
{
const std::string nested_type_json_str = R"(null)";
const std::string root_type_json_str = R"({ "a": 1, "nested": )" + nested_type_json_str + R"(, "anotherValue": "test" })";
auto expected = json({{"nested", nullptr}, {"anotherValue", "test"}, {"a", 1}});
auto filteredExpected = expected;
filteredExpected.erase("a");
SETUP_TESTCASES()
}
}
SECTION("with leading whitespace and newlines around root JSON")
{
const std::string initial_whitespace = R"(
)";
const std::string nested_type_json_str = R"({
"a": 1,
"nested": {
"b": "test"
},
"anotherValue": "test"
})";
const std::string end_whitespace = R"(
)";
const std::string root_type_json_str = initial_whitespace + nested_type_json_str + end_whitespace;
auto expected = json({{"a", 1}, {"nested", {{"b", "test"}}}, {"anotherValue", "test"}});
auto j = json::parse(root_type_json_str);
// 2. Check if the generated JSON is as expected
CHECK(j == expected);
// 3. Check if the start and end positions do not include the surrounding whitespace
CHECK(j.start_pos() == initial_whitespace.size());
CHECK(j.end_pos() == root_type_json_str.size() - end_whitespace.size());
}
}
#undef SETUP_TESTCASES
#endif
}
// this test relies on parse errors being thrown, so it is skipped when
tests/src/unit-class_parser_diagnostic_positions.cpp
-1960
View File
File diff suppressed because it is too large Load Diff
tests/src/unit-convenience.cpp
+4 -2
View File
@@ -98,8 +98,10 @@ void check_escaped(const char* original, const char* escaped = "", bool ensure_a
void check_escaped(const char* original, const char* escaped, const bool ensure_ascii)
{
std::stringstream ss;
json::serializer s(nlohmann::detail::output_adapter<char>(ss), ' ');
s.dump_escaped(original, ensure_ascii);
nlohmann::detail::output_stream_adapter<char> adapter(ss);
json::serializer s(adapter, ' ', false, ensure_ascii);
s.dump_escaped(original);
s.flush(); // dump_escaped writes into the serializer's internal buffer
CHECK(ss.str() == escaped);
}
} // namespace
tests/src/unit-diagnostic-positions-only.cpp
-44
View File
@@ -1,44 +0,0 @@
// __ _____ _____ _____
// __| | __| | | | JSON for Modern C++ (supporting code)
// | | |__ | | | | | | version 3.12.0
// |_____|_____|_____|_|___| https://github.com/nlohmann/json
//
// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
// SPDX-License-Identifier: MIT
#include "doctest_compatibility.h"
#ifdef JSON_DIAGNOSTICS
#undef JSON_DIAGNOSTICS
#endif
#define JSON_DIAGNOSTICS 0
#define JSON_DIAGNOSTIC_POSITIONS 1
#include <nlohmann/json.hpp>
using json = nlohmann::json;
TEST_CASE("Better diagnostics with positions only")
{
SECTION("invalid type")
{
const std::string json_invalid_string = R"(
{
"address": {
"street": "Fake Street",
"housenumber": "1"
}
}
)";
json j = json::parse(json_invalid_string);
CHECK_THROWS_WITH_AS(j.at("address").at("housenumber").get<int>(),
"[json.exception.type_error.302] (bytes 108-111) type must be number, but is string", json::type_error);
}
SECTION("invalid type without positions")
{
const json j = "foo";
CHECK_THROWS_WITH_AS(j.get<int>(),
"[json.exception.type_error.302] type must be number, but is string", json::type_error);
}
}
tests/src/unit-diagnostic-positions.cpp
+13 -1
View File
@@ -8,7 +8,9 @@
#include "doctest_compatibility.h"
#define JSON_DIAGNOSTICS 1
#ifndef JSON_DIAGNOSTICS
#define JSON_DIAGNOSTICS 1
#endif
#define JSON_DIAGNOSTIC_POSITIONS 1
#include <nlohmann/json.hpp>
@@ -27,8 +29,13 @@ TEST_CASE("Better diagnostics with positions")
}
)";
json j = json::parse(json_invalid_string);
#if JSON_DIAGNOSTICS
CHECK_THROWS_WITH_AS(j.at("address").at("housenumber").get<int>(),
"[json.exception.type_error.302] (/address/housenumber) (bytes 108-111) type must be number, but is string", json::type_error);
#else
CHECK_THROWS_WITH_AS(j.at("address").at("housenumber").get<int>(),
"[json.exception.type_error.302] (bytes 108-111) type must be number, but is string", json::type_error);
#endif
}
SECTION("invalid type without positions")
@@ -74,7 +81,12 @@ TEST_CASE("Better diagnostics with positions")
// (/foo/bar); the position of that parent is reported in the message
const json doc = json::parse(R"({"foo":{"bar":"a string"}})");
const json patch = json::parse(R"([{"op":"add","path":"/foo/bar/baz","value":1}])");
#if JSON_DIAGNOSTICS
CHECK_THROWS_WITH_AS(doc.patch(patch),
"[json.exception.out_of_range.411] (/foo/bar) (bytes 14-24) cannot add value: the JSON Patch 'add' target's parent is of type string, but must be an object or array", json::out_of_range);
#else
CHECK_THROWS_WITH_AS(doc.patch(patch),
"[json.exception.out_of_range.411] (bytes 14-24) cannot add value: the JSON Patch 'add' target's parent is of type string, but must be an object or array", json::out_of_range);
#endif
}
}
tests/src/unit-msgpack.cpp
+61
View File
@@ -1598,6 +1598,67 @@ TEST_CASE("MessagePack")
}
// use this testcase outside [hide] to run it with Valgrind
TEST_CASE("MessagePack nesting does not consume the call stack")
{
// Reading a container used to call back into the value reader once per
// element, so the native call stack grew with the nesting depth of the
// input: one frame per byte for repeated 0x91 (a one-element array), which
// crashes the process long before the input is exhausted (#5104). The
// containers are kept on a heap stack now.
//
// Note that deeply nested values must not be compared, copied or dumped
// here: those operations are still recursive, and would reintroduce the
// very crash this checks for. Depth is measured by descending instead.
SECTION("an unterminated chain is reported, not crashed on")
{
json _;
const std::vector<uint8_t> input(300000, 0x91);
CHECK_THROWS_WITH_AS(_ = json::from_msgpack(input), "[json.exception.parse_error.110] parse error at byte 300001: syntax error while parsing MessagePack value: unexpected end of input", json::parse_error&);
CHECK(json::from_msgpack(input, true, false).is_discarded());
}
SECTION("a well-formed deep value is read through the SAX interface")
{
std::vector<uint8_t> input(300000, 0x91);
input.push_back(0x01); // innermost value
SaxCountdown accept_all(600001);
CHECK(json::sax_parse(input, &accept_all, json::input_format_t::msgpack));
}
SECTION("a well-formed deep value is read into a value")
{
const std::size_t depth = 10000;
std::vector<uint8_t> input(depth, 0x91);
input.push_back(0x01);
json j = json::from_msgpack(input);
std::size_t measured = 0;
const json* p = &j;
while (p->is_array() && !p->empty())
{
p = &p->front();
++measured;
}
CHECK(measured == depth);
CHECK(p->is_number());
}
SECTION("containers are still read the same way")
{
CHECK(json::from_msgpack(std::vector<uint8_t>({0x90})) == json::array());
CHECK(json::from_msgpack(std::vector<uint8_t>({0x80})) == json::object());
CHECK(json::from_msgpack(std::vector<uint8_t>({0x92, 0x90, 0x80})) == json({json::array(), json::object()}));
CHECK(json::from_msgpack(std::vector<uint8_t>({0x91, 0x91, 0x91, 0x90})) == json({{{json::array()}}}));
CHECK(json::from_msgpack(std::vector<uint8_t>({0x81, 0xA1, 'a', 0x81, 0xA1, 'b', 0x92, 0x01, 0x02})) == json({{"a", {{"b", {1, 2}}}}}));
// array 16 and map 32, i.e. the counted forms
CHECK(json::from_msgpack(std::vector<uint8_t>({0xDC, 0x00, 0x02, 0x01, 0x02})) == json({1, 2}));
CHECK(json::from_msgpack(std::vector<uint8_t>({0xDF, 0x00, 0x00, 0x00, 0x01, 0xA1, 'k', 0xC3})) == json({{"k", true}}));
}
}
TEST_CASE("single MessagePack roundtrip")
{
SECTION("sample.json")
tests/src/unit-regression2.cpp
-910
View File
@@ -241,209 +241,6 @@ class my_allocator : public std::allocator<T>
};
};
/////////////////////////////////////////////////////////////////////
// for #3077
/////////////////////////////////////////////////////////////////////
class FooAlloc
{};
class Foo
{
public:
explicit Foo(const FooAlloc& /* unused */ = FooAlloc()) {}
bool value = false;
};
class FooBar
{
public:
Foo foo{}; // NOLINT(readability-redundant-member-init)
};
inline void from_json(const nlohmann::json& j, FooBar& fb) // NOLINT(misc-use-internal-linkage)
{
j.at("value").get_to(fb.foo.value);
}
/////////////////////////////////////////////////////////////////////
// for #3171
/////////////////////////////////////////////////////////////////////
struct for_3171_base // NOLINT(cppcoreguidelines-special-member-functions)
{
for_3171_base(const std::string& /*unused*/ = {}) {}
virtual ~for_3171_base();
for_3171_base(const for_3171_base& other) // NOLINT(hicpp-use-equals-default,modernize-use-equals-default)
: str(other.str)
{}
for_3171_base& operator=(const for_3171_base& other)
{
if (this != &other)
{
str = other.str;
}
return *this;
}
for_3171_base(for_3171_base&& other) noexcept
: str(std::move(other.str))
{}
for_3171_base& operator=(for_3171_base&& other) noexcept
{
if (this != &other)
{
str = std::move(other.str);
}
return *this;
}
virtual void _from_json(const json& j)
{
j.at("str").get_to(str);
}
std::string str{}; // NOLINT(readability-redundant-member-init)
};
for_3171_base::~for_3171_base() = default;
struct for_3171_derived : public for_3171_base
{
for_3171_derived() = default;
~for_3171_derived() override;
explicit for_3171_derived(const std::string& /*unused*/) { }
for_3171_derived(const for_3171_derived& other) // NOLINT(hicpp-use-equals-default,modernize-use-equals-default)
: for_3171_base(other)
{}
for_3171_derived& operator=(const for_3171_derived& other)
{
if (this != &other)
{
for_3171_base::operator=(other); // Call base class assignment operator
}
return *this;
}
for_3171_derived(for_3171_derived&& other) noexcept
: for_3171_base(std::move(other))
{}
for_3171_derived& operator=(for_3171_derived&& other) noexcept
{
if (this != &other)
{
for_3171_base::operator=(std::move(other)); // Call base class move assignment operator
}
return *this;
}
};
for_3171_derived::~for_3171_derived() = default;
inline void from_json(const json& j, for_3171_base& tb) // NOLINT(misc-use-internal-linkage)
{
tb._from_json(j);
}
/////////////////////////////////////////////////////////////////////
// for #3312
/////////////////////////////////////////////////////////////////////
#ifdef JSON_HAS_CPP_20
struct for_3312
{
std::string name;
};
inline void from_json(const json& j, for_3312& obj) // NOLINT(misc-use-internal-linkage)
{
j.at("name").get_to(obj.name);
}
#endif
/////////////////////////////////////////////////////////////////////
// for #3204
/////////////////////////////////////////////////////////////////////
struct for_3204_foo
{
for_3204_foo() = default;
explicit for_3204_foo(std::string /*unused*/) {} // NOLINT(performance-unnecessary-value-param)
};
struct for_3204_bar
{
enum constructed_from_t // NOLINT(cppcoreguidelines-use-enum-class)
{
constructed_from_none = 0,
constructed_from_foo = 1,
constructed_from_json = 2
};
explicit for_3204_bar(std::function<void(for_3204_foo)> /*unused*/) noexcept // NOLINT(performance-unnecessary-value-param)
: constructed_from(constructed_from_foo) {}
explicit for_3204_bar(std::function<void(json)> /*unused*/) noexcept // NOLINT(performance-unnecessary-value-param)
: constructed_from(constructed_from_json) {}
constructed_from_t constructed_from = constructed_from_none;
};
/////////////////////////////////////////////////////////////////////
// for #3333
/////////////////////////////////////////////////////////////////////
struct for_3333 final
{
for_3333(int x_ = 0, int y_ = 0) : x(x_), y(y_) {}
template <class T>
for_3333(const T& /*unused*/)
{
CHECK(false);
}
int x = 0;
int y = 0;
};
template <>
inline for_3333::for_3333(const json& j)
: for_3333(j.value("x", 0), j.value("y", 0))
{}
/////////////////////////////////////////////////////////////////////
// for #3810
/////////////////////////////////////////////////////////////////////
struct Example_3810
{
int bla{};
Example_3810() = default;
};
NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Example_3810, bla) // NOLINT(misc-use-internal-linkage)
/////////////////////////////////////////////////////////////////////
// for #4740
/////////////////////////////////////////////////////////////////////
#ifdef JSON_HAS_CPP_17
struct Example_4740
{
std::optional<std::string> host = std::nullopt;
std::optional<int> port = std::nullopt;
NLOHMANN_DEFINE_TYPE_INTRUSIVE_WITH_DEFAULT(Example_4740, host, port)
};
#endif
TEST_CASE("regression tests 2")
{
SECTION("issue #1001 - Fix memory leak during parser callback")
@@ -964,713 +761,6 @@ TEST_CASE("regression tests 2")
CHECK(j == k);
}
#if JSON_HAS_FILESYSTEM || JSON_HAS_EXPERIMENTAL_FILESYSTEM
// JSON_HAS_CPP_17 (do not remove; see note at top of file)
SECTION("issue #3070 - Version 3.10.3 breaks backward-compatibility with 3.10.2 ")
{
nlohmann::detail::std_fs::path text_path("/tmp/text.txt");
const json j(text_path);
const auto j_path = j.get<nlohmann::detail::std_fs::path>();
CHECK(j_path == text_path);
#if DOCTEST_CLANG || DOCTEST_GCC >= DOCTEST_COMPILER(8, 4, 0)
// only known to work on Clang and GCC >=8.4
CHECK_THROWS_WITH_AS(nlohmann::detail::std_fs::path(json(1)), "[json.exception.type_error.302] type must be string, but is number", json::type_error);
#endif
}
#endif
SECTION("issue #3077 - explicit constructor with default does not compile")
{
json j;
j[0]["value"] = true;
std::vector<FooBar> foo;
j.get_to(foo);
}
SECTION("issue #3108 - ordered_json doesn't support range based erase")
{
ordered_json j = {1, 2, 2, 4};
auto last = std::unique(j.begin(), j.end());
j.erase(last, j.end());
CHECK(j.dump() == "[1,2,4]");
j.erase(std::remove_if(j.begin(), j.end(), [](const ordered_json & val)
{
return val == 2;
}), j.end());
CHECK(j.dump() == "[1,4]");
}
SECTION("issue #3343 - json and ordered_json are not interchangeable")
{
json::object_t jobj({ { "product", "one" } });
ordered_json::object_t ojobj({{"product", "one"}});
auto jit = jobj.begin();
auto ojit = ojobj.begin();
CHECK(jit->first == ojit->first);
CHECK(jit->second.get<std::string>() == ojit->second.get<std::string>());
}
SECTION("issue #3171 - if class is_constructible from std::string wrong from_json overload is being selected, compilation failed")
{
const json j{{ "str", "value"}};
// failed with: error: no match for ‘operator=’ (operand types are ‘for_3171_derived’ and ‘const nlohmann::basic_json<>::string_t’
// {aka ‘const std::__cxx11::basic_string<char>’})
// s = *j.template get_ptr<const typename BasicJsonType::string_t*>();
auto td = j.get<for_3171_derived>();
CHECK(td.str == "value");
}
#ifdef JSON_HAS_CPP_20
SECTION("issue #3312 - Parse to custom class from unordered_json breaks on G++11.2.0 with C++20")
{
// see test for #3171
const ordered_json j = {{"name", "class"}};
for_3312 obj{};
j.get_to(obj);
CHECK(obj.name == "class");
}
#endif
#if defined(JSON_HAS_CPP_17) && JSON_USE_IMPLICIT_CONVERSIONS
SECTION("issue #3428 - Error occurred when converting nlohmann::json to std::any")
{
const json j;
const std::any a1 = j;
std::any&& a2 = j;
CHECK(a1.type() == typeid(j));
CHECK(a2.type() == typeid(j));
}
#endif
SECTION("issue #3204 - ambiguous regression")
{
const for_3204_bar bar_from_foo([](for_3204_foo) noexcept {}); // NOLINT(performance-unnecessary-value-param)
const for_3204_bar bar_from_json([](json) noexcept {}); // NOLINT(performance-unnecessary-value-param)
CHECK(bar_from_foo.constructed_from == for_3204_bar::constructed_from_foo);
CHECK(bar_from_json.constructed_from == for_3204_bar::constructed_from_json);
}
SECTION("issue #3333 - Ambiguous conversion from nlohmann::basic_json<> to custom class")
{
const json j
{
{"x", 1},
{"y", 2}
};
const for_3333 p = j;
CHECK(p.x == 1);
CHECK(p.y == 2);
}
SECTION("issue #3810 - ordered_json doesn't support construction from C array of custom type")
{
Example_3810 states[45]; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays)
// fix "not used" warning
states[0].bla = 1;
const auto* const expected = R"([{"bla":1},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0}])";
// This works:
nlohmann::json j;
j["test"] = states;
CHECK(j["test"].dump() == expected);
// This doesn't compile:
nlohmann::ordered_json oj;
oj["test"] = states;
CHECK(oj["test"].dump() == expected);
}
#ifdef JSON_HAS_CPP_17
SECTION("issue #4740 - build issue with std::optional")
{
const auto t1 = Example_4740();
const auto j1 = nlohmann::json(t1);
CHECK(j1.dump() == "{\"host\":null,\"port\":null}");
const auto t2 = j1.get<Example_4740>();
CHECK(!t2.host.has_value());
CHECK(!t2.port.has_value());
// improve coverage
auto t3 = Example_4740();
t3.port = 80;
t3.host = "example.com";
const auto j2 = nlohmann::json(t3);
CHECK(j2.dump() == "{\"host\":\"example.com\",\"port\":80}");
const auto t4 = j2.get<Example_4740>();
CHECK(t4.host.has_value());
CHECK(t4.port.has_value());
}
#endif
#if !defined(_MSVC_LANG)
// MSVC returns garbage on invalid enum values, so this test is excluded
// there.
SECTION("issue #4762 - json exception 302 with unhelpful explanation : type must be number, but is number")
{
// In #4762, the main issue was that a json object with an invalid type
// returned "number" as type_name(), because this was the default case.
// This test makes sure we now return "invalid" instead.
json j;
j.m_data.m_type = static_cast<json::value_t>(100); // NOLINT(clang-analyzer-optin.core.EnumCastOutOfRange)
CHECK(j.type_name() == "invalid");
}
#endif
#ifdef JSON_HAS_CPP_17
SECTION("issue #4804: from_cbor incompatible with std::vector<std::byte> as binary_t")
{
const std::vector<std::uint8_t> data = {0x80};
const auto decoded = json_4804::from_cbor(data);
CHECK((decoded == json_4804::array()));
}
SECTION("discussion #4209 - custom BinaryType direct assignment and round-tripping")
{
// Test that assigning a custom BinaryType directly creates a binary value, not an array
const std::vector<std::byte> original{std::byte{1}, std::byte{2}, std::byte{3}};
const json_4804 j = original;
CHECK(j.is_binary());
CHECK(!j.is_array());
// Test round-tripping: extracting the binary value back as the custom container type
const auto extracted = j.get<std::vector<std::byte>>();
CHECK(extracted == original);
// Test that the default json alias behavior is unchanged: std::vector<uint8_t> -> array
const json default_json = std::vector<std::uint8_t> {1, 2, 3};
CHECK(default_json.is_array());
CHECK(!default_json.is_binary());
}
SECTION("discussion #4209 - custom BinaryType extraction from parsed array")
{
// Test that extracting a custom BinaryType from a parsed JSON array still works
// (not just from a binary-typed node)
const auto j = json_4804::parse("[1,2,3]");
CHECK(j.is_array());
CHECK(!j.is_binary());
// Extracting as custom BinaryType should work from arrays
const auto extracted = j.get<std::vector<std::byte>>();
CHECK(extracted.size() == 3);
CHECK(extracted[0] == std::byte{1});
CHECK(extracted[1] == std::byte{2});
CHECK(extracted[2] == std::byte{3});
}
SECTION("issue #5046 - implicit conversion of return json to std::optional no longer implicit")
{
const json jval{};
auto GetValue = [](const json & valRoot) -> std::optional<json>
{
if (valRoot.contains("default"))
{
return valRoot.at("default");
}
return std::nullopt;
};
auto result = GetValue(jval);
CHECK(!result.has_value());
}
#endif
#if JSON_HAS_RANGES == 1
SECTION("issue #4440 - assert when using std::views::filter and GCC 10")
{
auto noOpFilter = std::views::filter([](auto&&) noexcept
{
return true;
});
json j = {1, 2, 3};
auto filtered = j | noOpFilter;
CHECK(*filtered.begin() == 1);
}
#endif
#if JSON_HAS_RANGES && !defined(__MINGW32__)
SECTION("issue #4916 - constructing array from C++20 ranges view does not work")
{
std::vector<int> nums{1, 2, 37, 42, 21};
auto filteredNums = nums | std::views::filter([](int i)
{
return i > 10;
});
json const j(filteredNums);
CHECK(j.type() == json::value_t::array);
CHECK(j == json({37, 42, 21}));
}
#endif
// owning_view is not available in libstdc++ < 12
#if JSON_HAS_RANGES && !defined(__MINGW32__) && !(defined(__GLIBCXX__) && _GLIBCXX_RELEASE < 12)
SECTION("issue #4916 - constructing array from prvalue C++20 ranges view (owning_view)")
{
json const j(std::vector<int> {1, 2, 37, 42, 21} | std::views::filter([](int i)
{
return i > 10;
}));
CHECK(j.type() == json::value_t::array);
CHECK(j == json({37, 42, 21}));
}
#endif
#if JSON_HAS_RANGES && !defined(__MINGW32__)
SECTION("issue #4916 - constructing array from C++20 transform view (prvalue elements)")
{
std::vector<int> nums{1, 2, 3};
auto t = nums | std::views::transform([](int i) noexcept
{
return i * 2;
});
json const j(t);
CHECK(j.type() == json::value_t::array);
CHECK(j == json({2, 4, 6}));
}
#endif
}
TEST_CASE_TEMPLATE("issue #4798 - nlohmann::json::to_msgpack() encode float NaN as double", T, double, float) // NOLINT(readability-math-missing-parentheses, bugprone-throwing-static-initialization)
{
// With issue #4798, we encode NaN, infinity, and -infinity as float instead
// of double to allow for smaller encodings.
const json jx = std::numeric_limits<T>::quiet_NaN();
const json jy = std::numeric_limits<T>::infinity();
const json jz = -std::numeric_limits<T>::infinity();
/////////////////////////////////////////////////////////////////////////
// MessagePack
/////////////////////////////////////////////////////////////////////////
// expected MessagePack values
const std::vector<std::uint8_t> msgpack_x = {{0xCA, 0x7F, 0xC0, 0x00, 0x00}};
const std::vector<std::uint8_t> msgpack_y = {{0xCA, 0x7F, 0x80, 0x00, 0x00}};
const std::vector<std::uint8_t> msgpack_z = {{0xCA, 0xFF, 0x80, 0x00, 0x00}};
CHECK(json::to_msgpack(jx) == msgpack_x);
CHECK(json::to_msgpack(jy) == msgpack_y);
CHECK(json::to_msgpack(jz) == msgpack_z);
CHECK(std::isnan(json::from_msgpack(msgpack_x).get<T>()));
CHECK(json::from_msgpack(msgpack_y).get<T>() == std::numeric_limits<T>::infinity());
CHECK(json::from_msgpack(msgpack_z).get<T>() == -std::numeric_limits<T>::infinity());
// Make sure the other MessagePakc encodings for NaN, infinity, and
// -infinity are still supported.
const std::vector<std::uint8_t> msgpack_x_2 = {{0xCB, 0x7F, 0xF8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}};
const std::vector<std::uint8_t> msgpack_y_2 = {{0xCB, 0x7F, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}};
const std::vector<std::uint8_t> msgpack_z_2 = {{0xCB, 0xFF, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}};
CHECK(std::isnan(json::from_msgpack(msgpack_x_2).get<T>()));
CHECK(json::from_msgpack(msgpack_y_2).get<T>() == std::numeric_limits<T>::infinity());
CHECK(json::from_msgpack(msgpack_z_2).get<T>() == -std::numeric_limits<T>::infinity());
/////////////////////////////////////////////////////////////////////////
// CBOR
/////////////////////////////////////////////////////////////////////////
// expected CBOR values
const std::vector<std::uint8_t> cbor_x = {{0xF9, 0x7E, 0x00}};
const std::vector<std::uint8_t> cbor_y = {{0xF9, 0x7C, 0x00}};
const std::vector<std::uint8_t> cbor_z = {{0xF9, 0xfC, 0x00}};
CHECK(json::to_cbor(jx) == cbor_x);
CHECK(json::to_cbor(jy) == cbor_y);
CHECK(json::to_cbor(jz) == cbor_z);
CHECK(std::isnan(json::from_cbor(cbor_x).get<T>()));
CHECK(json::from_cbor(cbor_y).get<T>() == std::numeric_limits<T>::infinity());
CHECK(json::from_cbor(cbor_z).get<T>() == -std::numeric_limits<T>::infinity());
// Make sure the other CBOR encodings for NaN, infinity, and -infinity are
// still supported.
const std::vector<std::uint8_t> cbor_x_2 = {{0xFA, 0x7F, 0xC0, 0x00, 0x00}};
const std::vector<std::uint8_t> cbor_y_2 = {{0xFA, 0x7F, 0x80, 0x00, 0x00}};
const std::vector<std::uint8_t> cbor_z_2 = {{0xFA, 0xFF, 0x80, 0x00, 0x00}};
const std::vector<std::uint8_t> cbor_x_3 = {{0xFB, 0x7F, 0xF8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}};
const std::vector<std::uint8_t> cbor_y_3 = {{0xFB, 0x7F, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}};
const std::vector<std::uint8_t> cbor_z_3 = {{0xFB, 0xFF, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}};
CHECK(std::isnan(json::from_cbor(cbor_x_2).get<T>()));
CHECK(json::from_cbor(cbor_y_2).get<T>() == std::numeric_limits<T>::infinity());
CHECK(json::from_cbor(cbor_z_2).get<T>() == -std::numeric_limits<T>::infinity());
CHECK(std::isnan(json::from_cbor(cbor_x_3).get<T>()));
CHECK(json::from_cbor(cbor_y_3).get<T>() == std::numeric_limits<T>::infinity());
CHECK(json::from_cbor(cbor_z_3).get<T>() == -std::numeric_limits<T>::infinity());
}
TEST_CASE("regression test #5074 - portable workaround for single-element brace init")
{
json const j_obj = {{"key", "value"}};
json const j = json::array({j_obj});
CHECK(j.is_array());
CHECK(j.size() == 1);
CHECK(j[0] == j_obj);
}
#if defined(JSON_BRACE_INIT_COPY_SEMANTICS) && (JSON_BRACE_INIT_COPY_SEMANTICS == 1)
TEST_CASE("regression test #5074 - single-element brace init with JSON_BRACE_INIT_COPY_SEMANTICS")
{
// with JSON_BRACE_INIT_COPY_SEMANTICS: single-element brace init copies/moves
json const j_obj = {{"key", "value"}, {"num", 42}};
json const j_arr = {1, 2, 3};
// object: brace init copies instead of wrapping
json const j1{j_obj};
CHECK(j1.is_object());
CHECK(j1 == j_obj);
// array: brace init copies instead of wrapping
json const j2{j_arr};
CHECK(j2.is_array());
CHECK(j2.size() == 3);
CHECK(j2 == j_arr);
// primitives still work as initializer lists
json const j3{true};
CHECK(j3.is_boolean());
json const j4{42};
CHECK(j4.is_number_integer());
}
#endif
struct Example_5122
{
float b = 2;
nlohmann::ordered_map<std::string, std::string> c{}; // NOLINT(readability-redundant-member-init): needed for GCC -Weffc++
int a = 1;
NLOHMANN_DEFINE_TYPE_INTRUSIVE_WITH_DEFAULT(Example_5122, b, c, a)
};
TEST_CASE("regression test #5122 - from_json into types holding nlohmann::ordered_map")
{
Example_5122 src;
src.c.emplace("first", "1");
src.c.emplace("second", "2");
ordered_json const j = src;
Example_5122 const dst = j.get<Example_5122>();
CHECK(dst.b == src.b);
CHECK(dst.a == src.a);
REQUIRE(dst.c.size() == src.c.size());
auto src_it = src.c.begin();
auto dst_it = dst.c.begin();
for (; src_it != src.c.end(); ++src_it, ++dst_it)
{
CHECK(dst_it->first == src_it->first);
CHECK(dst_it->second == src_it->second);
}
}
// -Wself-assign-overloaded was introduced in Clang 7. Gate the pragma on
// __has_warning so older Clang versions do not error with "unknown warning
// group". The __has_warning check has to stay inside the __clang__ branch
// because GCC does not provide it and would tokenize-error on the argument.
#if defined(__clang__) && defined(__has_warning)
#if __has_warning("-Wself-assign-overloaded")
DOCTEST_CLANG_SUPPRESS_WARNING_PUSH
DOCTEST_CLANG_SUPPRESS_WARNING("-Wself-assign-overloaded")
#endif
#endif
TEST_CASE("regression test #5122 - nlohmann::ordered_map copy-assignment is self-assignment safe")
{
nlohmann::ordered_map<std::string, std::string> m;
m.emplace("first", "1");
m.emplace("second", "2");
// Insertion order is preserved by ordered_map, so we can check it directly.
m = m;
REQUIRE(m.size() == 2);
auto it = m.begin();
CHECK(it->first == "first");
CHECK(it->second == "1");
++it;
CHECK(it->first == "second");
CHECK(it->second == "2");
}
#if defined(__clang__) && defined(__has_warning)
#if __has_warning("-Wself-assign-overloaded")
DOCTEST_CLANG_SUPPRESS_WARNING_POP
#endif
#endif
TEST_CASE("regression test #5122 - nlohmann::ordered_map move-assignment transfers contents")
{
nlohmann::ordered_map<std::string, std::string> src;
src.emplace("first", "1");
src.emplace("second", "2");
nlohmann::ordered_map<std::string, std::string> dst;
dst.emplace("stale", "x");
dst = std::move(src);
REQUIRE(dst.size() == 2);
auto it = dst.begin();
CHECK(it->first == "first");
CHECK(it->second == "1");
++it;
CHECK(it->first == "second");
CHECK(it->second == "2");
// Re-assigning into the moved-from object must leave it in a usable state.
src = nlohmann::ordered_map<std::string, std::string> {};
src.emplace("after-move", "3");
REQUIRE(src.size() == 1);
CHECK(src.begin()->first == "after-move");
}
// Stand-in for a third-party library (e.g., Eigen as of 3.4, which added
// STL-compatible begin()/end() to its vector types), living in its own
// namespace with its own to_json overload for its vector type.
namespace issue_4320_eigen
{
// "array-compatible" from the library's point of view (it has begin()/end()),
// but for which this (fake) third-party namespace provides its own to_json.
struct vector3
{
double v[3]; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays,cppcoreguidelines-use-default-member-init,modernize-use-default-member-init)
vector3(double x, double y, double z) : v{x, y, z} {} // NOLINT(hicpp-member-init,cppcoreguidelines-pro-type-member-init)
double x() const
{
return v[0];
}
double y() const
{
return v[1];
}
double z() const
{
return v[2];
}
double* begin()
{
return v;
}
double* end()
{
return v + 3;
}
const double* begin() const
{
return v;
}
const double* end() const
{
return v + 3;
}
};
inline void to_json(json& j, const vector3& v) // NOLINT(misc-use-internal-linkage)
{
j = {{"x", v.x()}, {"y", v.y()}, {"z", v.z()}};
}
} // namespace issue_4320_eigen
// The user's own namespace, using the (fake) Eigen type as an implementation
// detail behind a payload type that has nothing to do with vectors/arrays.
namespace issue_4320
{
// Publicly derives from issue_4320_eigen::vector3 but does *not* define its
// own to_json - it is only ever used as a temporary to reach the base
// class's to_json via ADL.
struct vector3_wrapper : issue_4320_eigen::vector3
{
using issue_4320_eigen::vector3::vector3;
};
struct payload
{
double x, y, z;
};
inline vector3_wrapper to_eigen(const payload& p) // NOLINT(misc-use-internal-linkage)
{
return {p.x, p.y, p.z};
}
inline void to_json(json& j, const payload& p) // NOLINT(misc-use-internal-linkage)
{
// Unqualified call, passing a *derived* vector3_wrapper: relies on ADL
// finding issue_4320_eigen::to_json(json&, const vector3&) through the
// vector3 base class, via a derived-to-base conversion. Must NOT resolve
// to the library's own generic array-compatible to_json (an exact-match
// template for vector3_wrapper, since it also has begin()/end()), which
// would serialize this as [x, y, z] instead of {"x":x, "y":y, "z":z}.
to_json(j, to_eigen(p));
}
} // namespace issue_4320
TEST_CASE("issue #4320 - custom base class must not leak nlohmann::detail into ADL")
{
// Before the fix, basic_json unconditionally derived from a type living in
// nlohmann::detail (json_default_base), which made nlohmann::detail an
// associated namespace of every basic_json for ADL purposes. That leaked
// the library's internal generic-array to_json overload into unqualified
// to_json() calls made from user code, silently bypassing user-defined
// to_json overloads reached via a derived-to-base conversion.
const issue_4320::payload p{1.0, 2.0, 3.0};
json j;
to_json(j, p);
CHECK(j == json({{"x", 1.0}, {"y", 2.0}, {"z", 3.0}}));
}
TEST_CASE("issue #5338 - truncated CBOR tagged binary subtype is rejected")
{
const std::vector<std::vector<std::uint8_t>> truncated_tags =
{
{0xD8},
{0xD9, 0x00},
{0xDA, 0x00, 0x00, 0x00},
{0xDB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}
};
for (const auto& data : truncated_tags)
{
CAPTURE(data);
for (const auto tag_handler :
{
json::cbor_tag_handler_t::ignore, json::cbor_tag_handler_t::store
})
{
CAPTURE(tag_handler);
const auto result = json::from_cbor(data, true, false, tag_handler);
CHECK(result.is_discarded());
}
}
}
TEST_CASE("issue #5402 - update(merge_objects=true) overwrites a primitive with an object")
{
json t = {{"k", 1}};
t.update(json{{"k", {{"x", 2}}}}, true);
CHECK(t == json({{"k", {{"x", 2}}}}));
json mixed = {{"keep", {{"a", 1}}}, {"replace", 1}};
mixed.update(json{{"keep", {{"b", 2}}}, {"replace", {{"x", 2}}}}, true);
CHECK(mixed == json({{"keep", {{"a", 1}, {"b", 2}}}, {"replace", {{"x", 2}}}}));
}
TEST_CASE("regression test - parser callback must not lose a duplicate key's prior value")
{
// a callback that rejects only the scalar value 2
const json::parser_callback_t drop_value_2 = [](int /*depth*/, json::parse_event_t ev, json & v) noexcept
{
return !(ev == json::parse_event_t::value && v == 2);
};
SECTION("duplicate key, second (scalar) value rejected - prior value is restored")
{
const json j = json::parse(R"({"a":1,"a":2})", drop_value_2);
CHECK(j.dump() == "{\"a\":1}");
}
SECTION("duplicate key, second value is an object rejected at object_end - prior value is restored")
{
const json j = json::parse(R"({"a":1,"a":{"x":2}})",
[](int depth, json::parse_event_t ev, json& /*parsed*/) noexcept
{
return !(ev == json::parse_event_t::object_end && depth == 1);
});
CHECK(j.dump() == "{\"a\":1}");
}
SECTION("duplicate key, second value is an array rejected at array_end - prior value is restored")
{
const json j = json::parse(R"({"a":1,"a":[9,9]})",
[](int depth, json::parse_event_t ev, json& /*parsed*/) noexcept
{
return !(ev == json::parse_event_t::array_end && depth == 1);
});
CHECK(j.dump() == "{\"a\":1}");
}
SECTION("duplicate key, second value accepted (scalar) - last value wins")
{
const json j = json::parse(R"({"a":1,"a":2})", [](int, json::parse_event_t, json&) noexcept
{
return true;
});
CHECK(j.dump() == "{\"a\":2}");
}
SECTION("duplicate key, second value accepted (object) - last value wins")
{
const json j = json::parse(R"({"a":1,"a":{"x":2}})", [](int, json::parse_event_t, json&) noexcept
{
return true;
});
CHECK(j.dump() == "{\"a\":{\"x\":2}}");
}
SECTION("brand new (non-duplicate) key, value rejected - member is fully absent")
{
const json j = json::parse(R"({"a":1,"b":2})", drop_value_2);
CHECK(j.dump() == "{\"a\":1}");
}
SECTION("duplicate key nested two levels deep")
{
const json j = json::parse(R"({"outer":{"a":1,"a":2}})", drop_value_2);
CHECK(j.dump() == "{\"outer\":{\"a\":1}}");
}
SECTION("three occurrences of the same key - middle rejected, last accepted")
{
const json j = json::parse(R"({"k":1,"k":2,"k":3})", drop_value_2);
CHECK(j.dump() == "{\"k\":3}");
}
}
TEST_CASE("regression test - excessive binary container size honors allow_exceptions=false")
{
// CBOR array with declared length 2^63
const std::vector<std::uint8_t> cbor = {0x9b, 0x80, 0, 0, 0, 0, 0, 0, 0};
// CBOR map with declared length 2^63
const std::vector<std::uint8_t> cbor_m = {0xbb, 0x80, 0, 0, 0, 0, 0, 0, 0};
// UBJSON array with declared length 2^63-1
const std::vector<std::uint8_t> ubj = {'[', '#', 'L', 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
// BJData array with declared length 2^63-1 (little endian)
const std::vector<std::uint8_t> bjd = {'[', '#', 'L', 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f};
// allow_exceptions=false must report failure instead of throwing/aborting
CHECK(json::from_cbor(cbor, true, false).is_discarded());
CHECK(json::from_cbor(cbor_m, true, false).is_discarded());
CHECK(json::from_ubjson(ubj, true, false).is_discarded());
CHECK(json::from_bjdata(bjd, true, false).is_discarded());
// allow_exceptions=true (the default) must still throw exactly as before.
// The exact message text is not checked here: on platforms where
// std::size_t is 32-bit, the CBOR reader's own length-narrowing check
// (get_cbor_container_size(), unrelated to this fix) intercepts a
// declared length of 2^63 before it ever reaches the check this test
// targets, with different (but equally valid, and already correct)
// wording -- see unit-cbor.cpp for coverage of that message.
json _;
CHECK_THROWS_AS(_ = json::from_cbor(cbor), json::out_of_range);
// regression guard: a genuinely truncated CBOR input must remain discarded
CHECK(json::from_cbor(std::vector<std::uint8_t> {0x9b, 0, 0, 0, 0, 0, 0, 0, 0x02}, true, false).is_discarded());
}
DOCTEST_CLANG_SUPPRESS_WARNING_POP
tests/src/unit-regression3.cpp
+1001
View File
File diff suppressed because it is too large Load Diff
tests/src/unit-serialization.cpp
+229
View File
@@ -387,3 +387,232 @@ TEST_CASE("dump for basic_json with long double number_float_t")
check_same(100.0L, 100.0);
}
}
TEST_CASE("serialization of strings (bulk fast path)")
{
// These cases exercise the SWAR bulk-copy fast path in dump_escaped and the
// internal write buffer: long runs, escapes interrupting runs, 0x7F/DEL,
// multibyte UTF-8 under both ensure_ascii settings, and payloads larger than
// the write buffer.
SECTION("long unescaped ASCII exceeds the write buffer")
{
const std::string big(3000, 'a');
const json j = big;
CHECK(j.dump() == '"' + big + '"');
CHECK(j.dump(-1, ' ', true) == '"' + big + '"');
// round-trips
CHECK(json::parse(j.dump()) == j);
}
SECTION("runs interrupted by escapes")
{
const json j = std::string(500, 'x') + "\n\"\\" + std::string(500, 'y');
const std::string out = j.dump();
CHECK(out == '"' + std::string(500, 'x') + "\\n\\\"\\\\" + std::string(500, 'y') + '"');
CHECK(json::parse(out) == j);
}
SECTION("DEL (0x7F) depends on ensure_ascii")
{
const json j = std::string("a\x7f" "b");
CHECK(j.dump(-1, ' ', false) == "\"a\x7f" "b\""); // copied verbatim
CHECK(j.dump(-1, ' ', true) == "\"a\\u007fb\""); // escaped
}
SECTION("multibyte UTF-8 under both ensure_ascii settings")
{
const json j = std::string("A\xc3\xa9\xe4\xbd\xa0\xf0\x9f\x98\x80Z"); // A é 你 😀 Z
// not escaping non-ASCII: bytes are copied through the bulk validator
CHECK(j.dump(-1, ' ', false) == "\"A\xc3\xa9\xe4\xbd\xa0\xf0\x9f\x98\x80Z\"");
// ensure_ascii: escaped (with a surrogate pair for the emoji)
CHECK(j.dump(-1, ' ', true) == "\"A\\u00e9\\u4f60\\ud83d\\ude00Z\"");
CHECK(json::parse(j.dump(-1, ' ', true)) == j);
}
SECTION("many small structural writes exceed the write buffer")
{
json arr = json::array();
for (int i = 0; i < 2000; ++i)
{
arr.push_back(i);
}
const std::string out = arr.dump();
CHECK(out.front() == '[');
CHECK(out.back() == ']');
CHECK(json::parse(out) == arr);
json obj = json::object();
for (int i = 0; i < 500; ++i)
{
obj["key" + std::to_string(i)] = i;
}
CHECK(json::parse(obj.dump()) == obj);
CHECK(json::parse(obj.dump(2)) == obj);
// an array of many empty strings emits a long run of single-character
// writes ('"', '"', ',') at shallow nesting depth, so the write buffer
// fills and flushes mid-run without the deep recursion that would
// overflow the stack on some debug builds
json many_empty = json::array();
for (int i = 0; i < 500; ++i)
{
many_empty.push_back("");
}
const std::string out2 = many_empty.dump();
CHECK(out2.size() > 1024); // spans multiple write-buffer flushes
CHECK(out2.front() == '[');
CHECK(out2.back() == ']');
CHECK(json::parse(out2) == many_empty);
}
SECTION("invalid UTF-8 handling is unaffected by the fast path")
{
const json j = std::string("valid\xff" "more");
CHECK_THROWS_WITH_AS(j.dump(), "[json.exception.type_error.316] invalid UTF-8 byte at index 5: 0xFF", json::type_error&);
CHECK(j.dump(-1, ' ', false, json::error_handler_t::replace) == "\"valid\xef\xbf\xbd" "more\"");
CHECK(j.dump(-1, ' ', true, json::error_handler_t::replace) == "\"valid\\ufffdmore\"");
CHECK(j.dump(-1, ' ', false, json::error_handler_t::ignore) == "\"validmore\"");
}
}
TEST_CASE("indentation is written straight into the write buffer")
{
// put_indent() memsets the indentation into the write buffer instead of
// copying it out of a pre-grown indentation string. These cases cover an
// indentation wider than the buffer, a non-space indentation character, and
// nesting deep enough that the accumulated indentation spans several
// buffer-fulls - the situations the old grow-a-string approach got wrong.
SECTION("indent_step wider than the write buffer")
{
const json j = {{"a", 1}};
// 2000 > the 1024-byte write buffer, and > the 512 the indentation
// string used to start at
CHECK(j.dump(2000) == "{\n" + std::string(2000, ' ') + "\"a\": 1\n}");
// several whole buffer-fulls, so the buffer is refilled once and then
// flushed repeatedly
CHECK(j.dump(5000) == "{\n" + std::string(5000, ' ') + "\"a\": 1\n}");
CHECK(j.dump(5000, '\t') == "{\n" + std::string(5000, '\t') + "\"a\": 1\n}");
// an exact multiple of the buffer size
CHECK(j.dump(4096) == "{\n" + std::string(4096, ' ') + "\"a\": 1\n}");
}
SECTION("a non-space indentation character is used throughout")
{
const json j = {{"a", 1}};
// 600 is past the point where the indentation used to be grown, which
// is where a hard-coded space would have shown up
CHECK(j.dump(600, '\t') == "{\n" + std::string(600, '\t') + "\"a\": 1\n}");
CHECK(j.dump(3, '.') == "{\n...\"a\": 1\n}");
}
SECTION("accumulated indentation spans several buffer-fulls")
{
// five levels deep at 400 per level: the innermost value is indented by
// 2000 characters, reached in steps that each straddle the buffer end
json j = json::array({1});
for (int i = 0; i < 4; ++i)
{
j = json::array({j});
}
const std::string out = j.dump(400);
CHECK(out.find(std::string("\n") + std::string(2000, ' ') + "1\n") != std::string::npos);
CHECK(json::parse(out) == j);
}
SECTION("indentation is unchanged for ordinary widths")
{
const json j = {{"a", {1, 2}}, {"b", nullptr}};
CHECK(j.dump(2) == "{\n \"a\": [\n 1,\n 2\n ],\n \"b\": null\n}");
CHECK(j.dump(0) == "{\n\"a\": [\n1,\n2\n],\n\"b\": null\n}");
}
}
TEST_CASE("serialization of deeply nested values")
{
// dump() descends into a bounded number of levels and writes out whatever
// is nested deeper than that without the call stack; see
// https://github.com/nlohmann/json/issues/5387
SECTION("nested deeper than the call stack could follow")
{
// parsing is iterative, so building these costs little
const std::size_t depth = 100000;
const std::string array_text = std::string(depth, '[') + '0' + std::string(depth, ']');
CHECK(json::parse(array_text).dump() == array_text);
std::string object_text;
object_text.reserve((6 * depth) + 1);
for (std::size_t i = 0; i < depth; ++i)
{
object_text += "{\"a\":";
}
object_text += '1';
object_text.append(depth, '}');
CHECK(json::parse(object_text).dump() == object_text);
}
SECTION("depths around the bound of the descent")
{
// Cover every depth around the bound, so that the two ways of writing a
// value are known to meet cleanly - wherever the bound is set.
for (std::size_t d = 1; d <= 300; ++d)
{
CAPTURE(d);
const std::string array_text = std::string(d, '[') + '7' + std::string(d, ']');
CHECK(json::parse(array_text).dump() == array_text);
std::string object_text;
for (std::size_t i = 0; i < d; ++i)
{
object_text += "{\"k\":";
}
object_text += '7';
object_text.append(d, '}');
CHECK(json::parse(object_text).dump() == object_text);
}
}
SECTION("pretty-printing across the bound")
{
for (std::size_t d = 120; d <= 140; ++d)
{
CAPTURE(d);
const json j = json::parse(std::string(d, '[') + '7' + std::string(d, ']'));
std::string expected;
for (std::size_t i = 0; i < d; ++i)
{
expected += std::string(2 * i, ' ') + "[\n";
}
expected += std::string(2 * d, ' ') + '7';
for (std::size_t i = d; i > 0; --i)
{
expected += '\n' + std::string(2 * (i - 1), ' ') + ']';
}
CHECK(j.dump(2) == expected);
}
}
SECTION("an empty container below the bound")
{
// an empty container is written out in full and never descended into,
// so it must not gain a newline when it is reached iteratively
for (std::size_t d = 125; d <= 135; ++d)
{
CAPTURE(d);
const std::string compact = std::string(d, '[') + "[]" + std::string(d, ']');
CHECK(json::parse(compact).dump() == compact);
const std::string with_object = std::string(d, '[') + "{}" + std::string(d, ']');
CHECK(json::parse(with_object).dump() == with_object);
}
}
}
tests/src/unit-ubjson.cpp
+166
View File
@@ -2149,6 +2149,172 @@ TEST_CASE("UBJSON")
}
}
TEST_CASE("UBJSON nesting does not consume the call stack")
{
// Containers used to be read by calling back into the value reader once
// per element, so the native call stack grew with the nesting depth of the
// input. '[' alone opens a container, so a payload of repeated '[' crashed
// the process (#5104), as did the optimized forms, which reach the same
// path through a type or size annotation. The containers are kept on a
// heap stack now.
//
// Deeply nested values must not be compared, copied or dumped here: those
// operations are still recursive and would reintroduce the crash.
json _;
SECTION("containers that end at a marker")
{
const std::vector<uint8_t> input(500000, '[');
CHECK_THROWS_WITH_AS(_ = json::from_ubjson(input), "[json.exception.parse_error.110] parse error at byte 500001: syntax error while parsing UBJSON value: unexpected end of input", json::parse_error&);
CHECK(json::from_ubjson(input, true, false).is_discarded());
}
SECTION("containers with a size")
{
std::vector<uint8_t> input;
for (std::size_t i = 0; i < 100000; ++i)
{
input.push_back('[');
input.push_back('#');
input.push_back('i');
input.push_back(1);
}
CHECK_THROWS_AS(_ = json::from_ubjson(input), json::parse_error&);
CHECK(json::from_ubjson(input, true, false).is_discarded());
}
SECTION("containers with a type and a size")
{
// '[' is a permitted optimized type in UBJSON, so each element of such
// a container is itself a container, read without a marker of its own
std::vector<uint8_t> input;
for (std::size_t i = 0; i < 100000; ++i)
{
const std::vector<uint8_t> level = {'[', '$', '[', '#', 'i', 1};
input.insert(input.end(), level.begin(), level.end());
}
CHECK_THROWS_AS(_ = json::from_ubjson(input), json::parse_error&);
CHECK(json::from_ubjson(input, true, false).is_discarded());
}
SECTION("a well-formed deep value is read through the SAX interface")
{
std::vector<uint8_t> input(100000, '[');
input.insert(input.end(), 100000, ']');
SaxCountdown accept_all(1000000);
CHECK(json::sax_parse(input, &accept_all, json::input_format_t::ubjson));
}
SECTION("a well-formed deep value is read into a value")
{
const std::size_t depth = 10000;
std::vector<uint8_t> input(depth, '[');
input.insert(input.end(), depth, ']');
json j = json::from_ubjson(input);
std::size_t measured = 0;
const json* p = &j;
while (p->is_array() && !p->empty())
{
p = &p->front();
++measured;
}
// the innermost array is empty, so the descent stops one level short
CHECK(measured == depth - 1);
}
SECTION("containers are still read the same way")
{
CHECK(json::from_ubjson(std::vector<uint8_t>({'[', ']'})) == json::array());
CHECK(json::from_ubjson(std::vector<uint8_t>({'{', '}'})) == json::object());
CHECK(json::from_ubjson(std::vector<uint8_t>({'[', '#', 'i', 0})) == json::array());
CHECK(json::from_ubjson(std::vector<uint8_t>({'{', '#', 'i', 0})) == json::object());
CHECK(json::from_ubjson(std::vector<uint8_t>({'[', '$', 'i', '#', 'i', 2, 1, 2})) == json({1, 2}));
CHECK(json::from_ubjson(std::vector<uint8_t>({'[', '#', 'i', 2, 'i', 1, 'i', 2})) == json({1, 2}));
CHECK(json::from_ubjson(std::vector<uint8_t>({'{', '$', 'i', '#', 'i', 1, 'i', 1, 'a', 1})) == json({{"a", 1}}));
// a no-op is not a value, so a container of them holds none
CHECK(json::from_ubjson(std::vector<uint8_t>({'[', '$', 'N', '#', 'i', 2})) == json::array());
// sized and unsized forms nested inside one another
CHECK(json::from_ubjson(std::vector<uint8_t>({'[', '[', '#', 'i', 2, 'i', 1, 'i', 2, ']'})) == json({{1, 2}}));
CHECK(json::from_ubjson(std::vector<uint8_t>({'[', '#', 'i', 1, '[', 'i', 1, ']'})) == json({{1}}));
// an optimized container of containers
CHECK(json::from_ubjson(std::vector<uint8_t>({'[', '$', '[', '#', 'i', 2, 'i', 1, ']', 'i', 2, ']'})) == json({{1}, {2}}));
}
SECTION("BJData containers are still read the same way")
{
// the ND-array wrapper and the binary shortcut are complete values,
// not containers the reader descends into
CHECK(json::from_bjdata(std::vector<uint8_t>({'[', '$', 'U', '#', '[', '$', 'i', '#', 'i', 2, 2, 3, 1, 2, 3, 4, 5, 6})) ==
json({{"_ArrayType_", "uint8"}, {"_ArraySize_", {2, 3}}, {"_ArrayData_", {1, 2, 3, 4, 5, 6}}}));
CHECK(json::from_bjdata(std::vector<uint8_t>({'[', '$', 'i', '#', 'i', 2, 1, 2})) == json({1, 2}));
CHECK(json::from_bjdata(std::vector<uint8_t>({'[', '[', 'i', 1, ']', ']'})) == json({{1}}));
}
}
TEST_CASE("UBJSON optimized arrays of a valueless type are bounded")
{
// An element of type 'Z', 'T' or 'F' is encoded by its marker alone, so an
// optimized array of one of those has no payload and the declared count is
// the only thing deciding how much is allocated. Ten bytes used to produce
// billions of values (#2793); every other type costs at least one byte per
// element and is bounded by the end of the input.
json _;
SECTION("an excessive count is rejected")
{
// 'l' is a big-endian int32: 0x7FFFFFFF elements, about 34 GB of value
for (const auto marker :
{'Z', 'T', 'F'
})
{
const std::vector<uint8_t> input = {'[', '$', static_cast<uint8_t>(marker), '#', 'l', 0x7F, 0xFF, 0xFF, 0xFF};
CHECK_THROWS_WITH_AS(_ = json::from_ubjson(input), "[json.exception.out_of_range.408] syntax error while parsing UBJSON size: excessive array size", json::out_of_range&);
CHECK(json::from_ubjson(input, true, false).is_discarded());
}
}
SECTION("ordinary counts are unaffected")
{
CHECK(json::from_ubjson(std::vector<uint8_t>({'[', '$', 'Z', '#', 'i', 3})) == json({nullptr, nullptr, nullptr}));
CHECK(json::from_ubjson(std::vector<uint8_t>({'[', '$', 'T', '#', 'i', 2})) == json({true, true}));
CHECK(json::from_ubjson(std::vector<uint8_t>({'[', '$', 'F', '#', 'i', 2})) == json({false, false}));
// 'N' is a no-op rather than a value, and still yields an empty array
CHECK(json::from_ubjson(std::vector<uint8_t>({'[', '$', 'N', '#', 'i', 2})) == json::array());
}
SECTION("a type with a payload is unaffected")
{
// A count past the limit is not rejected for 'U', which costs a byte
// per element and is bounded by the end of the input instead. The
// count is kept just past the limit rather than made huge, because a
// count that also exceeds the array's max_size() is reported as
// out_of_range before the input runs out, and max_size() depends on
// the width of std::size_t.
const std::vector<uint8_t> input = {'[', '$', 'U', '#', 'l', 0x00, 0x10, 0x00, 0x01};
CHECK_THROWS_WITH_AS(_ = json::from_ubjson(input), "[json.exception.parse_error.110] parse error at byte 10: syntax error while parsing UBJSON number: unexpected end of input", json::parse_error&);
CHECK(json::from_ubjson(input, true, false).is_discarded());
}
SECTION("the writer stays within what the reader accepts")
{
// below the limit the optimized form is used and is tiny; above it the
// writer falls back so that the result can still be read back
json const at_limit(1048576, nullptr);
const auto v_at_limit = json::to_ubjson(at_limit, true, true);
CHECK(v_at_limit.size() == 9);
CHECK(v_at_limit.at(1) == '$');
CHECK(json::from_ubjson(v_at_limit) == at_limit);
json const above_limit(1048577, nullptr);
const auto v_above_limit = json::to_ubjson(above_limit, true, true);
CHECK(v_above_limit.at(1) != '$');
CHECK(json::from_ubjson(v_above_limit) == above_limit);
}
}
TEST_CASE("Universal Binary JSON Specification Examples 1")
{
SECTION("Null Value")
tests/src/unit-user_defined_input.cpp
+239
View File
@@ -18,7 +18,12 @@
#include <nlohmann/json.hpp>
using nlohmann::json;
#include <array> // array
#include <cstddef> // size_t
#include <cstdint> // uint8_t
#include <list>
#include <string> // string
#include <vector> // vector
#if defined(__cpp_lib_concepts) && defined(JSON_HAS_CPP_20)
#include <iterator>
@@ -212,6 +217,66 @@ TEST_CASE("Parse with heterogeneous iterator and sentinel types")
CHECK(j2.at(0) == 1);
}
// A type whose data() hands out raw bytes but whose size() counts something
// else - here fixed-size records. Reading [data(), data() + size()) as bytes
// would silently truncate the input, so data() and size() alone must not be
// taken as evidence of contiguous byte storage.
struct record_buffer
{
using value_type = std::array<char, 4>;
std::string bytes;
const char* data() const noexcept
{
return bytes.data();
}
std::size_t size() const noexcept
{
return bytes.size() / sizeof(value_type);
}
const char* begin() const noexcept
{
return bytes.data();
}
const char* end() const noexcept
{
return bytes.data() + bytes.size();
}
};
TEST_CASE("Contiguous byte containers take the pointer adapter")
{
// Containers with contiguous single-byte storage are routed through the
// pointer-based adapter so the bulk fast paths apply in every standard, not
// only in C++20 where the library iterators model std::contiguous_iterator.
CHECK(nlohmann::detail::is_contiguous_byte_container<std::string>::value);
CHECK(nlohmann::detail::is_contiguous_byte_container<std::vector<char>>::value);
CHECK(nlohmann::detail::is_contiguous_byte_container<std::vector<std::uint8_t>>::value);
CHECK(nlohmann::detail::is_contiguous_byte_container<std::array<char, 4>>::value);
// input_adapter() takes its container by forwarding reference, so the trait
// is also asked about reference types
CHECK(nlohmann::detail::is_contiguous_byte_container<std::string&>::value);
CHECK(nlohmann::detail::is_contiguous_byte_container<const std::string&>::value);
// everything else keeps the iterator-based adapter
CHECK_FALSE(nlohmann::detail::is_contiguous_byte_container<std::list<char>>::value);
CHECK_FALSE(nlohmann::detail::is_contiguous_byte_container<std::vector<int>>::value);
CHECK_FALSE(nlohmann::detail::is_contiguous_byte_container<const char*>::value);
// including a type that has data() and size() but whose size() does not
// count the units data() points at: its value_type says so
CHECK_FALSE(nlohmann::detail::is_contiguous_byte_container<record_buffer>::value);
// and such a container still parses through its iterators, in full - taking
// it for a byte container would stop after data() + size() bytes
const record_buffer buffer{"[1,2,3,4,5]"};
CHECK(buffer.data() == buffer.bytes.data());
CHECK(buffer.size() * sizeof(record_buffer::value_type) < buffer.bytes.size());
CHECK(json::parse(buffer) == json({1, 2, 3, 4, 5}));
}
#if defined(__cpp_lib_concepts) && defined(JSON_HAS_CPP_20)
// JSON_HAS_CPP_20 (do not remove; see note at top of file)
TEST_CASE("Parse with std::counted_iterator and std::default_sentinel_t")
@@ -228,6 +293,180 @@ TEST_CASE("Parse with std::counted_iterator and std::default_sentinel_t")
const std::counted_iterator<iterator_type> first2(json_str.begin(), len);
CHECK(json::accept(first2, std::default_sentinel));
}
TEST_CASE("std::counted_iterator reaches the contiguous fast paths")
{
// A sized sentinel makes the remaining element count computable in O(1), so
// std::counted_iterator over a contiguous iterator must reach the same bulk
// string/number scanners as a plain pointer - not just the byte-at-a-time
// fallback (see #5268 for the equivalent memcpy fast path).
#if JSON_HAS_RANGES
// JSON_HAS_RANGES is 0 on standard libraries with an incomplete <ranges>
// (libstdc++ < 11, libc++ < 16), where the adapter deliberately falls back
// to the byte-at-a-time scanner; everything below still has to work there.
using adapter_type = nlohmann::detail::iterator_input_adapter<std::counted_iterator<const char*>, std::default_sentinel_t>;
CHECK(adapter_type::supports_bulk_scan);
CHECK(adapter_type::supports_seek);
#endif
// exercise every fast path: long ASCII run, multibyte UTF-8, escapes, and
// integer/floating-point numbers
const std::string json_str =
R"({"ascii":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",)"
"\"utf8\":\"\xe4\xb8\xad\xe6\x96\x87\xf0\x9f\x98\x80\xc3\xa9\","
R"("escaped":"aéb\n\\","ints":[0,-1,18446744073709551615,-9223372036854775808],)"
R"("floats":[1.5,-2.25e3,0.30000000000000004]})";
const auto len = static_cast<std::iter_difference_t<const char*>>(json_str.size());
const std::counted_iterator<const char*> first(json_str.data(), len);
const json j = json::parse(first, std::default_sentinel);
// parsing through the pointer adapter must give exactly the same result
CHECK(j == json::parse(json_str));
#if !defined(JSON_NOEXCEPTION)
// Diagnostics that quote the offending token are reconstructed from the
// already-consumed input (supports_seek), a path a sized sentinel only
// reaches now; check a few that include the "last read" text. Parsing
// invalid input aborts when exceptions are off, hence the guard.
// Raw strings and explicit bytes: an escaped literal and two literals
// written next to each other both read as mistakes to static analysis.
const auto byte = [](int value)
{
return std::string(1, static_cast<char>(value));
};
const std::vector<std::string> diagnostic_docs =
{
"1\nx",
"truX",
"[tru]",
R"("abc)",
R"(["\ud834"])",
R"(["a)" + byte(0x01) + R"(b"])",
R"([")" + byte(0xC3) + byte(0x28) + R"("])",
"[1e]",
R"(["aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaX)"
};
for (const auto& text : diagnostic_docs)
{
CAPTURE(text);
const std::counted_iterator<const char*> it(text.data(), static_cast<std::iter_difference_t<const char*>>(text.size()));
std::string counted_message;
std::string string_message;
try
{
const json counted_result = json::parse(it, std::default_sentinel);
static_cast<void>(counted_result);
}
catch (const json::parse_error& e)
{
counted_message = e.what();
}
try
{
const json string_result = json::parse(text);
static_cast<void>(string_result);
}
catch (const json::parse_error& e)
{
string_message = e.what();
}
CHECK_FALSE(counted_message.empty());
CHECK(counted_message == string_message);
}
// and errors must still be reported identically
const std::string bad = "[01\n]";
const std::counted_iterator<const char*> bad_first(bad.data(), static_cast<std::iter_difference_t<const char*>>(bad.size()));
std::string counted_what;
std::string string_what;
try
{
const json counted_result = json::parse(bad_first, std::default_sentinel);
static_cast<void>(counted_result);
}
catch (const json::parse_error& e)
{
counted_what = e.what();
}
try
{
const json string_result = json::parse(bad);
static_cast<void>(string_result);
}
catch (const json::parse_error& e)
{
string_what = e.what();
}
CHECK_FALSE(counted_what.empty());
CHECK(counted_what == string_what);
#endif
}
#if !defined(JSON_NOEXCEPTION)
// several cases below are truncated on purpose, and parsing invalid input
// aborts when exceptions are off
TEST_CASE("std::counted_iterator bulk scanning stops at the counted end")
{
// The count, not the size of the underlying buffer, is the end of the
// input: the bulk scanners must never look at the bytes behind it, even
// though they are readable. Each case is compared against parsing the
// equivalent prefix as a std::string.
const auto via_counted = [](const std::string & buf, std::size_t n) -> std::string
{
const std::counted_iterator<const char*> first(buf.data(), static_cast<std::iter_difference_t<const char*>>(n));
try
{
const json j = json::parse(first, std::default_sentinel);
return "OK|" + j.dump();
}
catch (const json::parse_error& e)
{
return {e.what()};
}
};
const auto via_prefix = [](const std::string & buf, std::size_t n) -> std::string
{
try
{
const json j = json::parse(buf.substr(0, n));
return "OK|" + j.dump();
}
catch (const json::parse_error& e)
{
return {e.what()};
}
};
struct testcase // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
{
const char* buffer;
std::size_t count;
};
const std::vector<testcase> cases =
{
{"[\"abc\"]____TRAILING____", 7}, // exact fit, tail hidden
{"[\"abcdefghijklmnop\"]____", 8}, // cut inside a string
{"[\"abc\"]____", 6}, // cut just before the closing quote
{"[12345]xxxxx", 4}, // cut inside a number
{"[123]999999", 5}, // number ends exactly at the count
{"[\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"]", 12}, // closing quote only behind the count
{"[\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"]", 19}, // cut inside an 8-byte SWAR stride
{"[\"\xe4\xb8\xad\xe6\x96\x87\"]", 5}, // cut inside a UTF-8 sequence
{"[\"\xe4\xb8\xad\xe6\x96\x87\"]____", 10}, // complete UTF-8, tail hidden
{"[1.25e3]TRAILINGDIGITS999", 7}, // number token reaches the count
};
for (const auto& tc : cases)
{
CAPTURE(tc.buffer);
CAPTURE(tc.count);
const std::string buffer = tc.buffer;
CHECK(via_counted(buffer, tc.count) == via_prefix(buffer, tc.count));
}
}
#endif
#endif
} // namespace