From 755bb16d7a80436646380b5f871b93c6a443a3b7 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Fri, 30 Oct 2026 16:48:31 +0100 Subject: [PATCH 01/54] Fix the red CI on develop: remove the duplicate is_editable in basic_json_view (#5966) #5919 replaced the bool template parameter Editable of basic_json_view by view_mode Mode and defined is_editable from Mode. #5890 was written before that and added a second is_editable = Editable to the private section, which no longer compiles ("duplicate member", "undeclared identifier Editable"). It broke every job that builds the json_view tests (macOS, MSVC, clang-cl, clang on Windows, the C++20 module job, CodeQL). Signed-off-by: Niels Lohmann --- include/nlohmann/json_view.hpp | 3 --- single_include/nlohmann/json_view.hpp | 3 --- 2 files changed, 6 deletions(-) diff --git a/include/nlohmann/json_view.hpp b/include/nlohmann/json_view.hpp index 0c6c7410b..9d26d12ea 100644 --- a/include/nlohmann/json_view.hpp +++ b/include/nlohmann/json_view.hpp @@ -766,9 +766,6 @@ class basic_json_view template friend class detail::view::view_side; friend iterator; - /// whether the view is one of an editable document - static constexpr bool is_editable = Editable; - basic_json_view(const document_data* d, const node* n) noexcept : m_doc(d), m_node(n) {} diff --git a/single_include/nlohmann/json_view.hpp b/single_include/nlohmann/json_view.hpp index c4636da14..8ff21c77f 100644 --- a/single_include/nlohmann/json_view.hpp +++ b/single_include/nlohmann/json_view.hpp @@ -9807,9 +9807,6 @@ class basic_json_view template friend class detail::view::view_side; friend iterator; - /// whether the view is one of an editable document - static constexpr bool is_editable = Editable; - basic_json_view(const document_data* d, const node* n) noexcept : m_doc(d), m_node(n) {} From 4fe670b5f4ce045a31bf108dca389755cb3b3563 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sat, 31 Oct 2026 13:54:02 +0100 Subject: [PATCH 02/54] json_view: decode chains of unicode escapes in a loop (#5969) After the first \uXXXX escape of a string, slow_string() now decodes the escapes that follow it directly, while the next six bytes are \u and four hex digits that are not a surrogate, instead of going round its main loop for each of them (that is how Python's json.dumps writes non-ASCII text by default). A surrogate, a bad digit, or fewer than six bytes leave the position at the backslash: the main loop handles them as before and reports the same error at the same position. twitterescaped.json (simdjson-data, 549 KB, 33,036 escapes), json_document::parse(), A/B in one binary, 6 code layouts, time after / before: - Apple M1 Max, Apple clang 21: 0.897 - x86-64 Haswell, GCC 16.2: 0.907 - x86-64 Haswell, Clang 23: 0.910 The other files of the benchmark set (twitter, citm_catalog, canada, gsoc-2018, apache_builds, instruments, tree-pretty, github_events, random, update-center, semanticscholar-corpus, twitter_timeline) stay within the noise of each machine. New test: chains of every length from 1 to 40 of escapes that give one, two, and three bytes of UTF-8, followed by the end of the string, text, escapes, surrogates, bad digits, and cut in every place, in keys and in values, against json::parse(). Signed-off-by: Niels Lohmann --- include/nlohmann/detail/view/builder.hpp | 15 +++++++ single_include/nlohmann/json_view.hpp | 15 +++++++ tests/src/unit-json_view_builder.cpp | 53 ++++++++++++++++++++++++ 3 files changed, 83 insertions(+) diff --git a/include/nlohmann/detail/view/builder.hpp b/include/nlohmann/detail/view/builder.hpp index 89b5af2ab..daf0d59bb 100644 --- a/include/nlohmann/detail/view/builder.hpp +++ b/include/nlohmann/detail/view/builder.hpp @@ -343,6 +343,21 @@ class builder } arena_ensure(w, w_end, 4); w = put_utf8(w, cp); + // A chain of unicode escapes ("\u3042\u3044\u3046", as Python's json.dumps writes non-ASCII text) is + // decoded here, without going round the loop above for each of them. Anything else (a surrogate, a + // bad digit, fewer than six bytes) leaves p at the backslash: the loop above handles it, and reports + // the same error at the same position as it would without this loop. + while (end - p >= 6 && p[0] == '\\' && p[1] == 'u') + { + const int next = hex_codepoint(p + 2); + if (next < 0 || (next & 0xF800) == 0xD800) + { + break; + } + p += 6; + arena_ensure(w, w_end, 4); + w = put_utf8(w, next); + } } else if (NLOHMANN_VIEW_LIKELY(d < 128 && simple_escape[d] != 0)) { diff --git a/single_include/nlohmann/json_view.hpp b/single_include/nlohmann/json_view.hpp index 8ff21c77f..e490f61cc 100644 --- a/single_include/nlohmann/json_view.hpp +++ b/single_include/nlohmann/json_view.hpp @@ -1585,6 +1585,21 @@ class builder } arena_ensure(w, w_end, 4); w = put_utf8(w, cp); + // A chain of unicode escapes ("\u3042\u3044\u3046", as Python's json.dumps writes non-ASCII text) is + // decoded here, without going round the loop above for each of them. Anything else (a surrogate, a + // bad digit, fewer than six bytes) leaves p at the backslash: the loop above handles it, and reports + // the same error at the same position as it would without this loop. + while (end - p >= 6 && p[0] == '\\' && p[1] == 'u') + { + const int next = hex_codepoint(p + 2); + if (next < 0 || (next & 0xF800) == 0xD800) + { + break; + } + p += 6; + arena_ensure(w, w_end, 4); + w = put_utf8(w, next); + } } else if (NLOHMANN_VIEW_LIKELY(d < 128 && simple_escape[d] != 0)) { diff --git a/tests/src/unit-json_view_builder.cpp b/tests/src/unit-json_view_builder.cpp index 4023c7848..7ac7a7d40 100644 --- a/tests/src/unit-json_view_builder.cpp +++ b/tests/src/unit-json_view_builder.cpp @@ -575,6 +575,59 @@ TEST_CASE("json_view builder: escapes between runs of text") } } +TEST_CASE("json_view builder: chains of unicode escapes") +{ + // Consecutive \uXXXX escapes are decoded in a loop of their own. Chains of + // every length from 1 to 40 of escapes that give one, two, and three + // bytes of UTF-8, in upper and lower case, followed by what ends a chain: + // the end of the string, text, another escape, UTF-8, and everything that + // is not a valid escape (a bad digit at every position of the last + // escape, a lone surrogate, a surrogate pair, a cut in every place). + const std::array chain = {{"\\u0041", "\\u00e4", "\\u00E4", "\\u20ac", "\\u3042", "\\uFFFF", "\\u0000", "\\u07ff"}}; + const std::array ends = {{"", "abc", "\\n", "\\u00e9", "\\ud83d\\ude00", "\\ud83d", "\\ude00", "\\ud83d\\u0041", "\\u12G4", "\\u12", "\xc3\xa4", "\\"}}; + std::mt19937 rng(2027); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed): reproducible + for (std::size_t length = 1; length <= 40; ++length) + { + for (int i = 0; i < 20; ++i) + { + std::string content = std::string(rng() % 20, 'a'); + for (std::size_t k = 0; k < length; ++k) + { + content += chain[rng() % chain.size()]; + } + const std::string whole = content; + for (const char* end : ends) + { + check_string(whole + end); + check_string(whole + end + std::string(rng() % 19, 'z')); + } + // a bad digit at every position of every escape near the end of the chain + for (std::size_t back = 1; back <= 6 && back <= length; ++back) + { + for (std::size_t digit = 2; digit < 6; ++digit) + { + std::string broken = whole; + broken[broken.size() - (6 * back) + digit] = "gG\"\x01 z"[rng() % 6]; + check_string(broken); + } + } + // the text cut in every place of the last three escapes + for (std::size_t cut = 0; cut <= 18 && cut <= whole.size(); ++cut) + { + check_string(whole.substr(0, whole.size() - cut)); + } + } + } + // a chain longer than the first growth of the arena, in a key and in a value + std::string long_chain; + for (int i = 0; i < 3000; ++i) + { + long_chain += chain[static_cast(i) % chain.size()]; + } + check_string(long_chain); + check_string(long_chain + "\\ud83d\\ude00" + long_chain); +} + TEST_CASE("json_view node integer bits") { using nlohmann::detail::view::integer_bits; From 367963f6bb6d9b7e066ef8365e82993f7fcc47d2 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sat, 31 Oct 2026 13:54:20 +0100 Subject: [PATCH 03/54] Test escaped object keys decoded through NLOHMANN_DEFINE_TYPE from a json_view (#5972) A json_view or json_document decodes a macro-defined struct directly and looks the members up by their decoded keys. Only plain keys were covered so far. The new section decodes structs defined with NLOHMANN_DEFINE_TYPE_INTRUSIVE, ..._WITH_DEFAULT and ..._WITH_NAMES from documents whose keys contain escape sequences, and compares the result with the conversion of the materialized value. It also pins the difference for a duplicate key written once plain and once escaped (the view takes the first member, the materialized value keeps the last one), and that an escaped backslash is part of the decoded key. Signed-off-by: Niels Lohmann --- tests/src/unit-json_view_from_json.cpp | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/src/unit-json_view_from_json.cpp b/tests/src/unit-json_view_from_json.cpp index 7e84a9081..a1d94d586 100644 --- a/tests/src/unit-json_view_from_json.cpp +++ b/tests/src/unit-json_view_from_json.cpp @@ -443,6 +443,30 @@ TEST_CASE("json_view from_json") CHECK(d.root().materialize().get().x == 3); } + SECTION("escaped keys: members are found by their decoded names") + { + // keys are decoded before they are compared, as by parse(); a lookup + // that compares the raw bytes of the source would miss these members + const json_document d = json_document::parse(R"([{"\u0078": 1, "y": 2}, {"V\u0061lue": 3, "Te\u0078t": "t"}, {"\u0062": "escaped"}, {"x": 1, "\u0078": 3, "y": 2}])"); + const json j = d.root().materialize(); + CHECK(d.root()[0].get().x == 1); + CHECK(d.root()[0].get().y == 2); + CHECK((d.root()[0].get() == j[0].get())); + CHECK(d.root()[1].get().value == 3); + CHECK(d.root()[1].get().text == "t"); + CHECK(d.root()[2].get().b == "escaped"); + CHECK(d.root()[2].get().a == 1); + // a duplicate key written once plain and once escaped: still the first member + CHECK(d.root()[3].get().x == 1); + CHECK(j[3].get().x == 3); +#if !defined(JSON_NOEXCEPTION) && !JSON_DIAGNOSTICS && !JSON_DIAGNOSTIC_POSITIONS + // an escaped backslash is part of the decoded key: "x\\" is not "x" + const json_document e = json_document::parse(R"({"x\\": 1, "y": 2})"); + CHECK(exception_of_call([&] { static_cast(e.root().get()); }) == exception_of_call([&] { static_cast(e.root().materialize().get()); })); + CHECK(exception_of_call([&] { static_cast(e.root().get()); }) == "[json.exception.out_of_range.403] key 'x' not found"); +#endif + } + SECTION("editable documents and ordered_json") { json_editable_document e = json_editable_document::parse(R"({"x": 1, "y": 2})"); From 9c349b1d7c393c6bc6dbbe4191e332e3bb4100b8 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sat, 31 Oct 2026 13:54:45 +0100 Subject: [PATCH 04/54] Write integers of 17 to 20 digits eight digits at a time in dump() (#5964) #5634 made dump() write integers below 10^16 eight digits at a time; larger ones still took the two-digits-per-division loop and count_digits(). Random 64-bit integers almost all have 17 to 20 digits (99.9 % of signed_ints.json and unsigned_ints.json), so Dump/signed_ints and Dump/unsigned_ints gained only 1.4x over 3.12.0, against 2.3x for small integers. A 64-bit magnitude is now written as its first one to four digits and two groups of eight. Wider magnitudes (__int128 and class types, #5805) keep the loop: number types of at most 64 bits are selected by type, so no comparison is reported as always true; for others the value is compared. The sign handling of #1447/#5815 (remove_sign() into number_magnitude_t) is unchanged. Signed-off-by: Niels Lohmann --- include/nlohmann/detail/output/serializer.hpp | 34 ++++++++++++++ single_include/nlohmann/json.hpp | 34 ++++++++++++++ tests/src/unit-serialization.cpp | 46 +++++++++++++++++++ 3 files changed, 114 insertions(+) diff --git a/include/nlohmann/detail/output/serializer.hpp b/include/nlohmann/detail/output/serializer.hpp index 598fda98a..4656dbdf1 100644 --- a/include/nlohmann/detail/output/serializer.hpp +++ b/include/nlohmann/detail/output/serializer.hpp @@ -1977,6 +1977,23 @@ class serializer return; } + // 17 to 20 digits: the first one to four, then twice eight + if (magnitude_fits_64_bits(abs_value, std::integral_constant())) + { + const auto value = static_cast(abs_value); + const std::uint64_t top = value / 10000000000000000u; // 1 to 1844 + const std::uint64_t low16 = value - (top * 10000000000000000u); + const std::uint64_t middle = low16 / 100000000u; + const std::uint64_t first = dtoa_impl::eight_digit_bytes(top); + const auto leading = static_cast(count_leading_zeros(first) / 8); // (first is not 0) + char* const p = buffer_ptr + n_chars; + dtoa_impl::store_msb_first(p, (first << (8 * leading)) + 0x3030303030303030u); + dtoa_impl::store_msb_first(p + 8 - leading, dtoa_impl::eight_digit_bytes(middle) + 0x3030303030303030u); + dtoa_impl::store_msb_first(p + 16 - leading, dtoa_impl::eight_digit_bytes(low16 - (middle * 100000000u)) + 0x3030303030303030u); + write_buffer_pos += n_chars + 24 - leading; + return; + } + n_chars += count_digits(abs_value); // spare 1 byte for '\0' @@ -2122,6 +2139,23 @@ class serializer } } + /// whether every number_magnitude_t fits 64 bits + static constexpr bool magnitude_has_at_most_64_bits = std::numeric_limits::is_specialized + && std::numeric_limits::digits <= 64; + + /// whether @a x fits 64 bits: always if number_magnitude_t has at most + /// 64 bits (selected by type, as the comparison would be reported as + /// always true), else compared + static bool magnitude_fits_64_bits(const number_magnitude_t& /*unused*/, std::true_type /*unused*/) noexcept + { + return true; + } + + static bool magnitude_fits_64_bits(const number_magnitude_t& x, std::false_type /*unused*/) noexcept + { + return x <= 18446744073709551615u; + } + /* * Helper function for dump_integer * diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 10b42dac1..83a7090cc 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -36224,6 +36224,23 @@ class serializer return; } + // 17 to 20 digits: the first one to four, then twice eight + if (magnitude_fits_64_bits(abs_value, std::integral_constant())) + { + const auto value = static_cast(abs_value); + const std::uint64_t top = value / 10000000000000000u; // 1 to 1844 + const std::uint64_t low16 = value - (top * 10000000000000000u); + const std::uint64_t middle = low16 / 100000000u; + const std::uint64_t first = dtoa_impl::eight_digit_bytes(top); + const auto leading = static_cast(count_leading_zeros(first) / 8); // (first is not 0) + char* const p = buffer_ptr + n_chars; + dtoa_impl::store_msb_first(p, (first << (8 * leading)) + 0x3030303030303030u); + dtoa_impl::store_msb_first(p + 8 - leading, dtoa_impl::eight_digit_bytes(middle) + 0x3030303030303030u); + dtoa_impl::store_msb_first(p + 16 - leading, dtoa_impl::eight_digit_bytes(low16 - (middle * 100000000u)) + 0x3030303030303030u); + write_buffer_pos += n_chars + 24 - leading; + return; + } + n_chars += count_digits(abs_value); // spare 1 byte for '\0' @@ -36369,6 +36386,23 @@ class serializer } } + /// whether every number_magnitude_t fits 64 bits + static constexpr bool magnitude_has_at_most_64_bits = std::numeric_limits::is_specialized + && std::numeric_limits::digits <= 64; + + /// whether @a x fits 64 bits: always if number_magnitude_t has at most + /// 64 bits (selected by type, as the comparison would be reported as + /// always true), else compared + static bool magnitude_fits_64_bits(const number_magnitude_t& /*unused*/, std::true_type /*unused*/) noexcept + { + return true; + } + + static bool magnitude_fits_64_bits(const number_magnitude_t& x, std::false_type /*unused*/) noexcept + { + return x <= 18446744073709551615u; + } + /* * Helper function for dump_integer * diff --git a/tests/src/unit-serialization.cpp b/tests/src/unit-serialization.cpp index 1173b851b..e2a41f272 100644 --- a/tests/src/unit-serialization.cpp +++ b/tests/src/unit-serialization.cpp @@ -23,6 +23,7 @@ using nlohmann::json; #include #include #include +#include #include #include @@ -248,6 +249,51 @@ TEST_CASE_TEMPLATE("serialization for extreme integer values", T, int32_t, uint3 } } +TEST_CASE("integers of every digit count are written like std::to_string") +{ + // dump_integer() writes up to 16 digits in two groups of eight and 17 to + // 20 digits as one to four digits and two groups of eight + std::vector values; + std::uint64_t power = 1; + for (int digits = 1; digits <= 20; ++digits) + { + values.push_back(power); // 1 followed by zeros + values.push_back(power + 1); + values.push_back((power * 2) - 1); + if (digits < 20) + { + values.push_back((power * 10) - 1); // all 9s + values.push_back((power * 10) - power); // 9 followed by zeros + power *= 10; + } + } + values.push_back((std::numeric_limits::max)()); + values.push_back(static_cast((std::numeric_limits::max)())); + values.push_back(static_cast((std::numeric_limits::max)()) + 1); + std::uint64_t state = 0x9E3779B97F4A7C15u; + for (int i = 0; i < 20000; ++i) + { + state = (state * 6364136223846793005u) + 1442695040888963407u; + values.push_back(state >> (state % 61)); // every bit length + } + + for (const auto v : values) + { + CAPTURE(v) + CHECK(json(v).dump() == std::to_string(v)); + if (v <= static_cast((std::numeric_limits::max)())) + { + const auto s = static_cast(v); + CHECK(json(s).dump() == std::to_string(s)); + CHECK(json(-s).dump() == std::to_string(-s)); + } + } + CHECK(json((std::numeric_limits::min)()).dump() == "-9223372036854775808"); + + // in a container, so the digits are followed by other output + CHECK(json::array({std::uint64_t{18446744073709551615u}, std::int64_t{-1234567890123456789}, 1}).dump() == "[18446744073709551615,-1234567890123456789,1]"); +} + TEST_CASE("dump with binary values") { auto binary = json::binary({1, 2, 3, 4}); From 3fd3e0fdaa54201e1b98cb523423bb548dcb0a38 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 1 Nov 2026 17:20:29 +0100 Subject: [PATCH 05/54] json_view: dump and read canada.json-style floats faster (batched float decisions) (#5908) Signed-off-by: Niels Lohmann --- include/nlohmann/detail/view/number.hpp | 171 ++++- include/nlohmann/detail/view/serializer.hpp | 404 ++++++++++- single_include/nlohmann/json_view.hpp | 576 ++++++++++++++- tests/src/unit-json_view_dump.cpp | 757 +++----------------- tests/src/unit-json_view_dump_escape.cpp | 298 ++++++++ tests/src/unit-json_view_large.cpp | 544 ++++++++++++++ 6 files changed, 2060 insertions(+), 690 deletions(-) create mode 100644 tests/src/unit-json_view_dump_escape.cpp create mode 100644 tests/src/unit-json_view_large.cpp diff --git a/include/nlohmann/detail/view/number.hpp b/include/nlohmann/detail/view/number.hpp index 10e2aa85d..7fc1b946f 100644 --- a/include/nlohmann/detail/view/number.hpp +++ b/include/nlohmann/detail/view/number.hpp @@ -19,6 +19,7 @@ #include #include #include +#include NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail @@ -103,12 +104,56 @@ NLOHMANN_VIEW_NOINLINE FloatType float_value(const char* first, const node& n) return convert_float(first, last, dot, mantissa_end); } +#if NLOHMANN_VIEW_VECTOR +/*! +@brief the value of the first 16 digits of a run of digits with a point after +int_digits of them (or none, if int_digits >= 16) + +The 17 bytes at p are read at once: the bytes after the point move down by one, +then pairs, quadruples, and octets of digits are combined by multiplications +(after simdjson). The bytes [p, p + 32) must be readable. Bytes that are not +digits (only in a damaged image loaded with image_check::bounds) give a wrong +value, but no undefined behavior. +*/ +NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t sixteen_digits(const unsigned char* p, unsigned int_digits) noexcept +{ +#if NLOHMANN_VIEW_NEON + const uint8x16_t low = vld1q_u8(p); + const uint8x16_t index = vcombine_u8(vcreate_u8(0x0706050403020100u), vcreate_u8(0x0F0E0D0C0B0A0908u)); + const uint8x16_t before_point = vcltq_u8(index, vdupq_n_u8(static_cast(int_digits))); + const uint8x16_t digits = vsubq_u8(vbslq_u8(before_point, low, vextq_u8(low, vld1q_u8(p + 16), 1)), vdupq_n_u8('0')); + const uint16x8_t twos = vpaddlq_u8(vmulq_u8(digits, vreinterpretq_u8_u16(vdupq_n_u16(0x010A)))); + const uint32x4_t fours = vpaddlq_u16(vmulq_u16(twos, vreinterpretq_u16_u32(vdupq_n_u32(0x00010064)))); + const uint64x2_t eights = vpaddlq_u32(vmulq_u32(fours, vreinterpretq_u32_u64(vdupq_n_u64(0x0000000100002710u)))); + return (vgetq_lane_u64(eights, 0) * 100000000u) + vgetq_lane_u64(eights, 1); +#else + // NOLINTBEGIN(portability-simd-intrinsics) + const __m128i low = _mm_loadu_si128(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) + const __m128i high = _mm_loadu_si128(static_cast(static_cast(p + 16))); // NOLINT(bugprone-casting-through-void) + const __m128i before_point = _mm_cmpgt_epi8(_mm_set1_epi8(static_cast(int_digits)), _mm_setr_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)); + const __m128i moved = _mm_or_si128(_mm_srli_si128(low, 1), _mm_slli_si128(high, 15)); + const __m128i digits = _mm_sub_epi8(_mm_or_si128(_mm_and_si128(before_point, low), _mm_andnot_si128(before_point, moved)), _mm_set1_epi8('0')); + // (16-bit lanes: SSE2 has no multiply-add of bytes) + const __m128i tens = _mm_setr_epi16(10, 1, 10, 1, 10, 1, 10, 1); + const __m128i twos = _mm_packs_epi32(_mm_madd_epi16(_mm_unpacklo_epi8(digits, _mm_setzero_si128()), tens), + _mm_madd_epi16(_mm_unpackhi_epi8(digits, _mm_setzero_si128()), tens)); + const __m128i fours = _mm_madd_epi16(twos, _mm_setr_epi16(100, 1, 100, 1, 100, 1, 100, 1)); + const __m128i eights = _mm_madd_epi16(_mm_packs_epi32(fours, fours), _mm_setr_epi16(10000, 1, 10000, 1, 10000, 1, 10000, 1)); + const auto upper = static_cast(_mm_cvtsi128_si32(eights)); + const auto lower = static_cast(_mm_cvtsi128_si32(_mm_srli_si128(eights, 4))); + // NOLINTEND(portability-simd-intrinsics) + return (std::uint64_t{upper} * 100000000u) + lower; +#endif +} +#endif + /*! @brief the digits of a float token with at most 19 digits, from its layout The digit layout recorded while parsing says where the integer digits, the fraction digits, and the exponent are, so the digits are read eight at a -time without scanning. +time without scanning (with SIMD, a token of 16 to 19 digits sixteen at a +time). @param[in] p first character of the token @param[in] e end of the token @@ -118,15 +163,34 @@ NLOHMANN_VIEW_ALWAYS_INLINE float_significand layout_decimal(const unsigned char { const bool negative = *p == '-'; p += negative ? 1 : 0; - std::uint64_t w = parse_upto19(p, int_digits, limit); - p += int_digits; + std::uint64_t w = 0; std::int64_t q = 0; - if (frac_digits != 0) +#if NLOHMANN_VIEW_VECTOR + const unsigned count = int_digits + frac_digits; + if (count >= 16 && limit - p >= 32) { - w = (w * int_pow10(frac_digits)) + parse_upto19(p + 1, frac_digits, limit); - p += 1 + frac_digits; + // (the point, if any, follows the integer digits; the digits after + // the first 16 are at most three) + w = sixteen_digits(p, int_digits); + for (unsigned i = 16; i < count; ++i) + { + w = (w * 10) + static_cast(p[i + (i >= int_digits ? 1 : 0)] - '0'); + } + p += count + (frac_digits != 0 ? 1 : 0); q = -static_cast(frac_digits); } + else /* NOLINT(readability/braces) */ +#endif + { + w = parse_upto19(p, int_digits, limit); + p += int_digits; + if (frac_digits != 0) + { + w = (w * int_pow10(frac_digits)) + parse_upto19(p + 1, frac_digits, limit); + p += 1 + frac_digits; + q = -static_cast(frac_digits); + } + } if (p != e) { // [eE][+-]digits; huge exponents saturate (the parser rejected @@ -161,12 +225,22 @@ NLOHMANN_VIEW_ALWAYS_INLINE float_significand layout_decimal(const unsigned char The result is correctly rounded by the lexer's conversion (detail::decimal_to_float(): Clinger's fast path where both operands are exact, else the Eisel-Lemire algorithm, which needs no fallback for up to 19 -digits), so it is the value parse() produces. +digits), so it is the value parse() produces. Tokens of 16 or more digits go +to Eisel-Lemire directly: the fast path takes few of them (none of 17 digits), +and its check is a branch that data like GeoJSON coordinates mispredicts. */ template NLOHMANN_VIEW_ALWAYS_INLINE FloatType layout_float(const unsigned char* p, const unsigned char* e, unsigned int_digits, unsigned frac_digits, const unsigned char* limit) noexcept { - return decimal_to_float(layout_decimal(p, e, int_digits, frac_digits, limit)); + const float_significand d = layout_decimal(p, e, int_digits, frac_digits, limit); + if (int_digits + frac_digits >= 16) + { + using format = ieee_binary_format::digits>; + native_float_t value{}; + float_from_bits(eisel_lemire(d.exponent, d.w) | (d.negative ? (std::uint64_t{1} << format::sign_bit()) : 0u), value); + return value; + } + return decimal_to_float(d); } /// the value of a float set by an edit: its token (the shortest round-trip @@ -185,6 +259,80 @@ NLOHMANN_VIEW_NOINLINE FloatType edited_float(const char* token, const node& n) return float_value(token, n); } +#if NLOHMANN_VIEW_VECTOR +/// eisel_lemire() for the token w * 10^q of coordinate_float() +template +NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t coordinate_bits(std::int64_t q, std::uint64_t w, Format /*format*/) noexcept +{ + return eisel_lemire(q, w); +} + +/*! +@brief (binary64) eisel_lemire() without the cases that tokens of 16 or 17 +digits with 5 or more after the point cannot reach: a value of 0, a +subnormal, an infinity, or a tie (q < -4) +*/ +NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t coordinate_bits(std::int64_t q, std::uint64_t w, ieee_binary_format<53> format) noexcept +{ + if (NLOHMANN_VIEW_UNLIKELY(w == 0 || q > -5)) + { + return eisel_lemire>(q, w); + } + static_cast(format); + const int lz = count_leading_zeros(w); + const std::uint64_t normalized = w << static_cast(lz); + const auto index = static_cast(2 * (q - pow5_128_smallest_power)); + const uint128_parts product = full_multiplication(normalized, pow5_128()[index]); + std::uint64_t high = product.high; + if (NLOHMANN_VIEW_UNLIKELY((high & 0x1FFu) == 0x1FFu)) + { + // (the lower bits may carry into the result) + const std::uint64_t second = full_multiplication(normalized, pow5_128()[index + 1]).high; + high += product.low + second < product.low ? 1u : 0u; + } + const auto upperbit = static_cast(high >> 63u); + std::uint64_t mantissa = high >> (upperbit + 9); + auto power2 = static_cast(((217706 * q) >> 16) + 63 + static_cast(upperbit) - lz + 1023); + mantissa = (mantissa + (mantissa & 1u)) >> 1u; + const bool carry = mantissa >= (std::uint64_t{2} << 52u); + mantissa = carry ? (std::uint64_t{1} << 52u) : mantissa; + power2 += carry ? 1u : 0u; + return (mantissa & ~(std::uint64_t{1} << 52u)) | (power2 << 52u); +} + +/*! +@brief (binary32 and binary64) the value of a float token of the source with +16 or 17 digits, a point, and no exponent, the shape of most coordinates +written with full precision; whether the token of n has this shape + +These tokens skip the general layout of layout_decimal(): no digits beyond +the 17th, no exponent (and Eisel-Lemire directly, as in layout_float()). +*/ +template +NLOHMANN_VIEW_ALWAYS_INLINE bool coordinate_float(const document_data& d, const node& n, FloatType& value) noexcept +{ + const unsigned int_digits = n.extra & 0xFFu; + const unsigned frac_digits = n.extra >> 8u; + const unsigned count = int_digits + frac_digits; + const auto* const first = reinterpret_cast(d.src + n.off); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + const unsigned negative = first[0] == '-' ? 1u : 0u; + if (count - 16u > 1u || frac_digits == 0 || n.len != negative + count + 1 || d.size - n.off < 33) + { + return false; + } + const unsigned char* const digits = first + negative; + const std::uint64_t w = sixteen_digits(digits, int_digits); + // (the 17th digit follows the point) + const std::uint64_t w17 = (w * 10) + static_cast(digits[17] - '0'); + using format = ieee_binary_format::digits>; + native_float_t result{}; + float_from_bits(coordinate_bits(-static_cast(frac_digits), count == 17 ? w17 : w, format{}) | (static_cast(negative) << format::sign_bit()), result); + value = result; + return true; +} + +#endif + /// the value of the float token of a node, as parse() converts it; floats and /// doubles with at most 19 digits are converted from the digit layout template @@ -200,6 +348,13 @@ FloatType float_value(const document_data& d, const node& n) template FloatType float_value(const document_data& d, const node& n, std::true_type /*binary32 or binary64*/) { +#if NLOHMANN_VIEW_VECTOR + FloatType value {}; + if (coordinate_float(d, n, value)) + { + return value; + } +#endif const unsigned int_digits = n.extra & 0xFFu; const unsigned frac_digits = n.extra >> 8u; if (NLOHMANN_VIEW_LIKELY(int_digits + frac_digits <= 19)) // (255 marks "many") diff --git a/include/nlohmann/detail/view/serializer.hpp b/include/nlohmann/detail/view/serializer.hpp index b41ac683f..2bfa2d2c0 100644 --- a/include/nlohmann/detail/view/serializer.hpp +++ b/include/nlohmann/detail/view/serializer.hpp @@ -14,7 +14,9 @@ #include // size_t #include // uint8_t, uint32_t #include // memcpy, memset +#include // less #include // numeric_limits +#include // unique_ptr #include // integral_constant #include // vector @@ -384,6 +386,143 @@ struct depth_counter } }; +/*! +@brief where the double x nearest to a token w * 10^q lies, for 10^15 <= w < +10^17 and -17 <= q <= -1 (Eisel-Lemire, then a fixed-point position) + +dump() writes the shortest decimal in the rounding interval of x (the closest +one if there are several, see zmij::to_shortest()). Its digits are those of a +multiple of 10^g of the token's last digit, with g = dec_exp + 1 - q for x's +decimal exponent dec_exp: the multiple of 10^g nearest to x if it lies in the +interval (at most one does), else the multiple of 10^(g - 1) nearest to x +(see token_digits_1() and token_digits_2()). For g <= 0, the token itself is +in the interval. + +The position is exact in a 64-bit fixed-point number with 56 bits after the +point (of the token's last digit): x / 10^q = 2c * 5^-q / 2^t for x = c * 2^e2 +and t = q + 1 - e2, with at most 56 bits after the point here, and |x / 10^q - +w| is at most half the spacing of the doubles, below 12 units. So only the low +64 bits of 2c * 5^-q are needed. c and e2 come from Eisel-Lemire's product +directly (as eisel_lemire() rounds it: no tie is possible for q < -4, and its +second product is added where it can carry into the result). + +@param[out] offset x / 10^q - w + 32, with 56 bits after the point +@param[out] radius half the spacing of the doubles at x, likewise (+ 1 if + the interval is closed) +@return g; 3 if the digits are not decided here (a power of two, whose + interval is not symmetric, or out of range) +*/ +NLOHMANN_VIEW_ALWAYS_INLINE int locate_token(std::uint64_t w, int q, std::uint64_t& offset, std::uint64_t& radius) noexcept +{ + const int lz = count_leading_zeros(w); + const std::uint64_t normalized = w << static_cast(lz); + const auto index = static_cast(2 * (q - pow5_128_smallest_power)); + const uint128_parts product = full_multiplication(normalized, pow5_128()[index]); + std::uint64_t high = product.high; + if (NLOHMANN_VIEW_UNLIKELY((high & 0x1FFu) == 0x1FFu)) + { + // (as eisel_lemire(): the lower bits may carry into the result) + const std::uint64_t second = full_multiplication(normalized, pow5_128()[index + 1]).high; + high += product.low + second < product.low ? 1u : 0u; + } + const auto upperbit = static_cast(high >> 63u); + const std::uint64_t mantissa = high >> (upperbit + 9); + const std::uint64_t c = (mantissa + (mantissa & 1u)) >> 1u; + // e2 as eisel_lemire() computes the exponent (log2(10) ~ 217706 / 2^16) + const int e2 = static_cast((217706 * static_cast(q)) >> 16) + 63 + static_cast(upperbit) - lz - 52; + const int t = q + 1 - e2; + const int g = ((e2 * 315653) >> 20) + 1 - q; // (dec_exp as in zmij::to_shortest(), for a regular x) + static const std::array fives = {{1u, 5u, 25u, 125u, 625u, 3125u, 15625u, 78125u, 390625u, 1953125u, 9765625u, 48828125u, 244140625u, 1220703125u, 6103515625u, 30517578125u, 152587890625u}}; + const std::uint64_t five = fives[static_cast(-q)]; // 5^-q + const unsigned s = static_cast(56 - t) & 63u; + offset = ((2 * c * five) << s) - (w << 56u) + (std::uint64_t{32} << 56u); + radius = (five << s) + ((c & 1u) == 0 ? 1u : 0u); + // (c = 2^52: a power of two; c = 2^53: rounded up to one. With 8 to 16 + // fraction digits, as the batch's tokens have, t is in [15, 40] and g at + // most 2. x lies halfway between two candidates only if its distance from + // the token is a whole unit (g == 2) or a half (g == 1): such tokens are + // left to the full conversion) + const bool decided = c - ((std::uint64_t{1} << 52u) + 1) < (std::uint64_t{1} << 52u) - 1 + && (offset & ((std::uint64_t{1} << 55u) - 1)) != 0; + return decided ? g : 3; +} + +/// all bits set if b, else none +NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t mask_if(bool b) noexcept +{ + return std::uint64_t{0} - static_cast(b); +} + +/// a if b, else c, without a branch (the conditions of token_digits_1/2() are +/// as good as random, and GCC compiled their ?: as branches: one misprediction +/// per float of canada.json on x86-64) +NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t select_if(bool b, std::uint64_t a, std::uint64_t c) noexcept +{ + return c ^ ((a ^ c) & mask_if(b)); +} + +/// the distance of a position (see locate_token()) from the biased point 32 +NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t biased_distance(std::uint64_t position) noexcept +{ + constexpr std::uint64_t center = std::uint64_t{32} << 56u; + // |position - center|, without a branch: negated (complemented, plus 1) if below + const std::uint64_t below = mask_if(position < center); + return ((position - center) ^ below) - below; +} + +/*! +@brief (g == 1) the digits dump() writes for a token, from the position of its +double (see locate_token()): the multiple of 10 nearest to x if it is in the +interval, else w rounded to x (an exact tie between two candidates, like +754570942.33984373, whose double is 754570942.33984375, is excluded by +locate_token()) + +With |x - w| at most the radius, which is below half the spacing of the +candidates, the multiple of 10 nearest to x is w - r10 or the one above it. +*/ +NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t token_digits_1(std::uint64_t w, std::uint64_t offset, std::uint64_t radius, std::uint64_t r10) noexcept +{ + constexpr std::uint64_t one = std::uint64_t{1} << 56u; + const std::uint64_t from = offset + (r10 * one); // x - (w - r10) (+ 32) + const bool up = from > 37 * one; // (nearer to the multiple above) + const bool shorter = biased_distance(from - (mask_if(up) & (10 * one))) < radius; + const std::uint64_t a = w - r10 + (mask_if(up) & 10u); + const std::uint64_t b = w + ((offset + (one / 2)) >> 56u) - 32; // w rounded to x + return select_if(shorter, a, b); +} + +/*! +@brief (g == 2) the digits dump() writes for a token, as token_digits_1(): the +multiple of 100 nearest to x if it is in the interval, else the multiple of 10 +nearest to x +*/ +NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t token_digits_2(std::uint64_t w, std::uint64_t offset, std::uint64_t radius, std::uint64_t r10, std::uint64_t r100) noexcept +{ + constexpr std::uint64_t one = std::uint64_t{1} << 56u; + const std::uint64_t from = offset + (r100 * one); + const bool up = from > 82 * one; + const bool shorter = biased_distance(from - (mask_if(up) & (100 * one))) < radius; + const std::uint64_t a = w - r100 + (mask_if(up) & 100u); + // x - (w - r10), in [-12, 21]: the multiple of 10 nearest to x is that one, + // or one or two above, or one below + const std::uint64_t y = offset + (r10 * one); + const std::uint64_t steps = static_cast(y > 37 * one) + static_cast(y > 47 * one); + const bool down = y < 27 * one; + const std::uint64_t b = w - r10 + (10 * steps) - (mask_if(down) & 10u); + return select_if(shorter, a, b); +} + +/// the four ASCII digits of x < 10^4, the first in the lowest byte (as +/// little-endian memory holds them) +NLOHMANN_VIEW_ALWAYS_INLINE std::uint32_t four_digit_text(std::uint32_t x) noexcept +{ + const std::uint32_t ab = (x * 5243u) >> 19u; // x / 100 + std::uint32_t v = ab | ((x - (ab * 100u)) << 16u); + const std::uint32_t a = ((v * 103u) >> 10u) & 0x000F000Fu; // / 10 per 16-bit lane + v = a | ((v - (a * 10u)) << 8u); + return v + 0x30303030u; +} + /// how the view's dump() writes a value struct dump_style { @@ -711,6 +850,7 @@ class view_serializer std::array buffer; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read std::vector heap; small_stack stack(buffer.data(), buffer.size(), heap); + float_batch batch; const char* const src = m_doc.src; const char* const src_end = src + m_doc.size; char* w = m_out.cursor(); @@ -850,7 +990,7 @@ class view_serializer else if (enabled(fast_double::value)) { room(64); - w = write_double_at(w, *n); + w = write_double_ahead(w, src, n, batch); } else { @@ -875,6 +1015,20 @@ class view_serializer case value_t::array: { const bool object = n->kind == node_kind::object; + // (read-only documents, compact output) a pair of floats, + // like the coordinates of GeoJSON, without a frame + if (enabled(!Editable && !Pretty && !SourceNumbers && fast_double::value) && n->next == 3 && n->len == 2 + && n[1].kind == node_kind::number_float && n[2].kind == node_kind::number_float) + { + room(2 * 64); + *w++ = '['; + w = write_double_ahead(w, src, n + 1, batch); + *w++ = ','; + w = write_double_ahead(w, src, n + 2, batch); + *w++ = ']'; + n += 2; // (the element before the next node) + break; + } room(8); if (n->len == 0) { @@ -984,6 +1138,254 @@ class view_serializer } } + /// (float_batch) the mark of digits whose rounding carried beyond the + /// token's last four (see mark()) + static constexpr std::uint64_t carried() noexcept + { + return std::uint64_t{1} << 63u; + } + + /// floats of 16 or 17 digits ahead of the walk, converted together (see + /// convert_ahead()), with their digits as mark() keeps them + struct float_batch + { + static constexpr std::size_t capacity() noexcept + { + return 256; + } + /// (on the heap, allocated by the first conversion: a frame of more + /// than a page made every dump of a small document slower) + struct entries + { + std::array at; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::array digits; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + }; + std::unique_ptr storage{}; + const node** at = nullptr; + std::uint64_t* digits = nullptr; + std::size_t count = 0; + std::size_t next = 0; + const node* scanned = nullptr; ///< where the last scan of the node array ended + }; + + /// (doubles) the float at n as dump() writes it, at w (64 bytes of room): + /// from the batch if it holds n + NLOHMANN_VIEW_ALWAYS_INLINE char* write_double_ahead(char* w, const char* src, const node* n, float_batch& batch) + { +#if NLOHMANN_VIEW_VECTOR && NLOHMANN_VIEW_LITTLE_ENDIAN + // (digits 0: not written by the batch; carried(): rounding carried + // beyond the last four digits) + if (batch.next != batch.count && batch.at[batch.next] == n && batch.digits[batch.next] - 1 < carried() - 1) + { + return write_token_digits(w, src, *n, batch.digits[batch.next++]); + } + return write_double_beside(w, n, batch); +#else + static_cast(src); + static_cast(batch); + return write_double_at(w, *n); +#endif + } + +#if NLOHMANN_VIEW_VECTOR && NLOHMANN_VIEW_LITTLE_ENDIAN + /// write_double_ahead() for a float that is not the batch's next one (out + /// of line: the walk is faster with less code in its loop) + NLOHMANN_VIEW_NOINLINE char* write_double_beside(char* w, const node* n, float_batch& batch) + { + // (an editable document's walk can skip nodes of the array) + const std::less before{}; // NOLINT(modernize-use-transparent-functors) C++11 has no std::less<> + while (batch.next != batch.count && before(batch.at[batch.next], n)) + { + ++batch.next; + } + if ((batch.next != batch.count && batch.at[batch.next] == n) || convert_ahead(n, batch)) + { + const std::uint64_t digits = batch.digits[batch.next++]; + if ((digits & carried()) != 0) + { + return write_carried_digits(w, *n, digits & ~carried()); + } + if (digits != 0) + { + return write_token_digits(w, m_doc.src, *n, digits); + } + } + return write_double_at(w, *n); + } + + /*! + @brief convert the floats of 16 or 17 digits among the next nodes of the + array at once; whether n is the first of them + + Such tokens (like the coordinates of GeoJSON, written with 17 digits) are + written as the token with its last four digits replaced + (write_token_digits()). Each step (the digits, locate_token(), + token_digits_1/2()) is a long chain of dependent operations; one loop per + step over many floats lets the processor overlap the floats, which it does + not across the steps of one float (on Apple M1, one loop for the digits and + locate_token() made the dump of canada.json 12 % slower). Each node of the array + is scanned once; a node that the walk does not reach in order (in an + editable document) is written without the batch. + */ + NLOHMANN_VIEW_NOINLINE bool convert_ahead(const node* n, float_batch& batch) + { + const std::less before{}; // NOLINT(modernize-use-transparent-functors) C++11 has no std::less<> + const node* const last = m_doc.nodes + m_doc.node_count; + if (before(n, batch.scanned) || before(n, m_doc.nodes) || !before(n, last)) + { + return false; + } + const char* const src = m_doc.src; + const char* const src_end = src + m_doc.size; + const node* const stop = last - n > 1024 ? n + 1024 : last; + batch.count = 0; + batch.next = 0; + // the digits of each token (and its last four from the text) + std::array digits; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::array exponent; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::array last_four; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + const node* p = n; + if (!batch.storage) + { + batch.storage.reset(new typename float_batch::entries); // NOLINT(cppcoreguidelines-owning-memory,modernize-make-unique): C++11 + batch.at = batch.storage->at.data(); + batch.digits = batch.storage->digits.data(); + } + for (; p != stop && batch.count != float_batch::capacity(); ++p) + { + // a token of the source: 16 or 17 digits, at least 8 after the + // point, no exponent, 32 bytes to read + const unsigned frac_digits = p->extra >> 8u; + const unsigned int_digits = p->extra & 0xFFu; + const unsigned count = int_digits + frac_digits; + if (p->kind == node_kind::number_float && (p->flags & node_flags::storage) == 0 + && count - 16u <= 1u && frac_digits >= 8 && src_end - (src + p->off) >= 33) + { + const auto* const token = reinterpret_cast(src + p->off); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + const unsigned sign = token[0] == '-' ? 1 : 0; + const unsigned char* const first = token + sign; + const std::uint64_t w = sixteen_digits(first, int_digits); + // (the 17th digit follows the point; a token with an exponent + // gets digits outside the range of locate_token(), and is not + // written by the batch. A token 0.xxx with 16 significant + // digits is: its double is written as "0." and its digits too) + const std::uint64_t w17 = (w * 10) + static_cast(first[17] - '0'); + const std::size_t i = batch.count++; + batch.at[i] = p; + if (p->len != sign + count + 1) + { + digits[i] = 0; + } + else + { + digits[i] = count == 17 ? w17 : w; + } + exponent[i] = -static_cast(frac_digits); + // (byte 0: the last digit, byte 1: the last two, bits 16-29: the last four) + std::uint32_t four = 0; + std::memcpy(&four, token + p->len - 4, 4); + four -= 0x30303030u; + const std::uint32_t pairs = ((four & 0x0F0F0F0Fu) * 2561u) >> 8u; // (byte 0, 2: the pairs) + last_four[i] = (four >> 24u) | ((pairs >> 16u & 0xFFu) << 8u) | ((((pairs & 0x00FF00FFu) * 6553601u) >> 16u) << 16u); + } + } + batch.scanned = p; + // the position of each double, and the floats of either kind + std::array offset; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::array radius; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::array kind_1; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::array kind_2; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::size_t count_1 = 0; + std::size_t count_2 = 0; + for (std::size_t i = 0; i != batch.count; ++i) + { + const std::uint64_t w = digits[i]; + // (the range excludes the values of bytes that are not digits, only + // in a damaged image loaded with image_check::bounds) + const int g = w - 1000000000000000u < 99000000000000000u ? locate_token(w, exponent[i], offset[i], radius[i]) : 3; + batch.digits[i] = mask_if(g <= 0) & ((last_four[i] >> 16u) + 1); + kind_1[count_1] = static_cast(i); + count_1 += static_cast(g == 1); + kind_2[count_2] = static_cast(i); + count_2 += static_cast(g == 2); + } + // the digits to write, in one loop per kind (see mark()) + for (std::size_t k = 0; k != count_1; ++k) + { + const std::size_t i = kind_1[k]; + const std::uint64_t s = token_digits_1(digits[i], offset[i], radius[i], last_four[i] & 0xFFu); + batch.digits[i] = mark(digits[i], s, last_four[i] >> 16u); + } + for (std::size_t k = 0; k != count_2; ++k) + { + const std::size_t i = kind_2[k]; + const std::uint64_t s = token_digits_2(digits[i], offset[i], radius[i], last_four[i] & 0xFFu, (last_four[i] >> 8u) & 0xFFu); + batch.digits[i] = mark(digits[i], s, last_four[i] >> 16u); + } + return batch.count != 0 && batch.at[0] == n; + } + + /*! + @brief a float token of 16 or 17 digits as dump() writes it, at w (64 bytes + of room), from the digits of its double (see locate_token()) whose + rounding carried beyond the token's last four digits + + (like 43.42027299999999 -> 43.420273): the digits go to the writer of + Zmij's results as its 15 or 16 leading digits and one more, which omits + the trailing zeros. + */ + NLOHMANN_VIEW_NOINLINE char* write_carried_digits(char* w, const node& n, std::uint64_t digits) const noexcept + { + const int exponent = -(n.extra >> 8u); + const std::uint64_t leading = digits / 10; + const auto last = static_cast(digits - (leading * 10)); + const bool seventeen = digits >= 10000000000000000u; + *w = '-'; + w += m_doc.src[n.off] == '-' ? 1 : 0; + return ::nlohmann::detail::dtoa_impl::write_shortest(w, seventeen ? ::nlohmann::detail::zmij::shortest_decimal{leading, exponent, last, last != 0} + : ::nlohmann::detail::zmij::shortest_decimal{digits, exponent - 1, 0, false}); + } + + /// the digits s of a token with the digits w and the last four digits + /// r4, as the batch keeps them: the new last four digits + 1 (only those + /// are written over the token's), or s marked if it differs from w beyond + /// them (0 stays for a float the batch does not write) + static std::uint64_t mark(std::uint64_t w, std::uint64_t s, std::uint64_t r4) noexcept + { + const std::uint64_t last = s - (w - r4); + return last < 10000 ? last + 1 : s | carried(); + } + + /*! + @brief a float token of 16 or 17 digits as dump() writes it, at w (64 bytes + of room): the token with its last four digits replaced by digits - 1 (see + mark(); the others are the token's), and without trailing zeros after the + point, but one + */ + char* write_token_digits(char* w, const char* src, const node& n, std::uint64_t digits) + { + const char* const token = src + n.off; + std::memcpy(w, token, 32); + char* const end = w + n.len; + const std::uint32_t last = four_digit_text(static_cast(digits - 1)); + std::memcpy(end - 4, &last, 4); + // (the token's text before the new digits, from the source: reading + // the output back would wait for the stores) + std::uint64_t before = 0; + std::memcpy(&before, token + n.len - 12, 8); + const std::uint32_t last_zeros = last ^ 0x30303030u; + const std::uint64_t before_zeros = before ^ 0x3030303030303030u; + if (NLOHMANN_VIEW_UNLIKELY(last_zeros == 0 && before_zeros == 0)) + { + return write_double_at(w, n); // (twelve or more trailing zeros) + } + const unsigned zeros = last_zeros != 0 ? (static_cast(count_leading_zeros(last_zeros)) / 8) - 4 + : 4 + (static_cast(count_leading_zeros(before_zeros)) / 8); + const unsigned frac_digits = n.extra >> 8u; + return end - (zeros < frac_digits ? zeros : frac_digits - 1); + } +#endif + /// a float node as dump() writes it void write_float_node(const node& n) { diff --git a/single_include/nlohmann/json_view.hpp b/single_include/nlohmann/json_view.hpp index e490f61cc..7e20544ba 100644 --- a/single_include/nlohmann/json_view.hpp +++ b/single_include/nlohmann/json_view.hpp @@ -2537,6 +2537,8 @@ NLOHMANN_JSON_NAMESPACE_END // #include +// #include + NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail @@ -2621,12 +2623,56 @@ NLOHMANN_VIEW_NOINLINE FloatType float_value(const char* first, const node& n) return convert_float(first, last, dot, mantissa_end); } +#if NLOHMANN_VIEW_VECTOR +/*! +@brief the value of the first 16 digits of a run of digits with a point after +int_digits of them (or none, if int_digits >= 16) + +The 17 bytes at p are read at once: the bytes after the point move down by one, +then pairs, quadruples, and octets of digits are combined by multiplications +(after simdjson). The bytes [p, p + 32) must be readable. Bytes that are not +digits (only in a damaged image loaded with image_check::bounds) give a wrong +value, but no undefined behavior. +*/ +NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t sixteen_digits(const unsigned char* p, unsigned int_digits) noexcept +{ +#if NLOHMANN_VIEW_NEON + const uint8x16_t low = vld1q_u8(p); + const uint8x16_t index = vcombine_u8(vcreate_u8(0x0706050403020100u), vcreate_u8(0x0F0E0D0C0B0A0908u)); + const uint8x16_t before_point = vcltq_u8(index, vdupq_n_u8(static_cast(int_digits))); + const uint8x16_t digits = vsubq_u8(vbslq_u8(before_point, low, vextq_u8(low, vld1q_u8(p + 16), 1)), vdupq_n_u8('0')); + const uint16x8_t twos = vpaddlq_u8(vmulq_u8(digits, vreinterpretq_u8_u16(vdupq_n_u16(0x010A)))); + const uint32x4_t fours = vpaddlq_u16(vmulq_u16(twos, vreinterpretq_u16_u32(vdupq_n_u32(0x00010064)))); + const uint64x2_t eights = vpaddlq_u32(vmulq_u32(fours, vreinterpretq_u32_u64(vdupq_n_u64(0x0000000100002710u)))); + return (vgetq_lane_u64(eights, 0) * 100000000u) + vgetq_lane_u64(eights, 1); +#else + // NOLINTBEGIN(portability-simd-intrinsics) + const __m128i low = _mm_loadu_si128(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) + const __m128i high = _mm_loadu_si128(static_cast(static_cast(p + 16))); // NOLINT(bugprone-casting-through-void) + const __m128i before_point = _mm_cmpgt_epi8(_mm_set1_epi8(static_cast(int_digits)), _mm_setr_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)); + const __m128i moved = _mm_or_si128(_mm_srli_si128(low, 1), _mm_slli_si128(high, 15)); + const __m128i digits = _mm_sub_epi8(_mm_or_si128(_mm_and_si128(before_point, low), _mm_andnot_si128(before_point, moved)), _mm_set1_epi8('0')); + // (16-bit lanes: SSE2 has no multiply-add of bytes) + const __m128i tens = _mm_setr_epi16(10, 1, 10, 1, 10, 1, 10, 1); + const __m128i twos = _mm_packs_epi32(_mm_madd_epi16(_mm_unpacklo_epi8(digits, _mm_setzero_si128()), tens), + _mm_madd_epi16(_mm_unpackhi_epi8(digits, _mm_setzero_si128()), tens)); + const __m128i fours = _mm_madd_epi16(twos, _mm_setr_epi16(100, 1, 100, 1, 100, 1, 100, 1)); + const __m128i eights = _mm_madd_epi16(_mm_packs_epi32(fours, fours), _mm_setr_epi16(10000, 1, 10000, 1, 10000, 1, 10000, 1)); + const auto upper = static_cast(_mm_cvtsi128_si32(eights)); + const auto lower = static_cast(_mm_cvtsi128_si32(_mm_srli_si128(eights, 4))); + // NOLINTEND(portability-simd-intrinsics) + return (std::uint64_t{upper} * 100000000u) + lower; +#endif +} +#endif + /*! @brief the digits of a float token with at most 19 digits, from its layout The digit layout recorded while parsing says where the integer digits, the fraction digits, and the exponent are, so the digits are read eight at a -time without scanning. +time without scanning (with SIMD, a token of 16 to 19 digits sixteen at a +time). @param[in] p first character of the token @param[in] e end of the token @@ -2636,15 +2682,34 @@ NLOHMANN_VIEW_ALWAYS_INLINE float_significand layout_decimal(const unsigned char { const bool negative = *p == '-'; p += negative ? 1 : 0; - std::uint64_t w = parse_upto19(p, int_digits, limit); - p += int_digits; + std::uint64_t w = 0; std::int64_t q = 0; - if (frac_digits != 0) +#if NLOHMANN_VIEW_VECTOR + const unsigned count = int_digits + frac_digits; + if (count >= 16 && limit - p >= 32) { - w = (w * int_pow10(frac_digits)) + parse_upto19(p + 1, frac_digits, limit); - p += 1 + frac_digits; + // (the point, if any, follows the integer digits; the digits after + // the first 16 are at most three) + w = sixteen_digits(p, int_digits); + for (unsigned i = 16; i < count; ++i) + { + w = (w * 10) + static_cast(p[i + (i >= int_digits ? 1 : 0)] - '0'); + } + p += count + (frac_digits != 0 ? 1 : 0); q = -static_cast(frac_digits); } + else /* NOLINT(readability/braces) */ +#endif + { + w = parse_upto19(p, int_digits, limit); + p += int_digits; + if (frac_digits != 0) + { + w = (w * int_pow10(frac_digits)) + parse_upto19(p + 1, frac_digits, limit); + p += 1 + frac_digits; + q = -static_cast(frac_digits); + } + } if (p != e) { // [eE][+-]digits; huge exponents saturate (the parser rejected @@ -2679,12 +2744,22 @@ NLOHMANN_VIEW_ALWAYS_INLINE float_significand layout_decimal(const unsigned char The result is correctly rounded by the lexer's conversion (detail::decimal_to_float(): Clinger's fast path where both operands are exact, else the Eisel-Lemire algorithm, which needs no fallback for up to 19 -digits), so it is the value parse() produces. +digits), so it is the value parse() produces. Tokens of 16 or more digits go +to Eisel-Lemire directly: the fast path takes few of them (none of 17 digits), +and its check is a branch that data like GeoJSON coordinates mispredicts. */ template NLOHMANN_VIEW_ALWAYS_INLINE FloatType layout_float(const unsigned char* p, const unsigned char* e, unsigned int_digits, unsigned frac_digits, const unsigned char* limit) noexcept { - return decimal_to_float(layout_decimal(p, e, int_digits, frac_digits, limit)); + const float_significand d = layout_decimal(p, e, int_digits, frac_digits, limit); + if (int_digits + frac_digits >= 16) + { + using format = ieee_binary_format::digits>; + native_float_t value{}; + float_from_bits(eisel_lemire(d.exponent, d.w) | (d.negative ? (std::uint64_t{1} << format::sign_bit()) : 0u), value); + return value; + } + return decimal_to_float(d); } /// the value of a float set by an edit: its token (the shortest round-trip @@ -2703,6 +2778,80 @@ NLOHMANN_VIEW_NOINLINE FloatType edited_float(const char* token, const node& n) return float_value(token, n); } +#if NLOHMANN_VIEW_VECTOR +/// eisel_lemire() for the token w * 10^q of coordinate_float() +template +NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t coordinate_bits(std::int64_t q, std::uint64_t w, Format /*format*/) noexcept +{ + return eisel_lemire(q, w); +} + +/*! +@brief (binary64) eisel_lemire() without the cases that tokens of 16 or 17 +digits with 5 or more after the point cannot reach: a value of 0, a +subnormal, an infinity, or a tie (q < -4) +*/ +NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t coordinate_bits(std::int64_t q, std::uint64_t w, ieee_binary_format<53> format) noexcept +{ + if (NLOHMANN_VIEW_UNLIKELY(w == 0 || q > -5)) + { + return eisel_lemire>(q, w); + } + static_cast(format); + const int lz = count_leading_zeros(w); + const std::uint64_t normalized = w << static_cast(lz); + const auto index = static_cast(2 * (q - pow5_128_smallest_power)); + const uint128_parts product = full_multiplication(normalized, pow5_128()[index]); + std::uint64_t high = product.high; + if (NLOHMANN_VIEW_UNLIKELY((high & 0x1FFu) == 0x1FFu)) + { + // (the lower bits may carry into the result) + const std::uint64_t second = full_multiplication(normalized, pow5_128()[index + 1]).high; + high += product.low + second < product.low ? 1u : 0u; + } + const auto upperbit = static_cast(high >> 63u); + std::uint64_t mantissa = high >> (upperbit + 9); + auto power2 = static_cast(((217706 * q) >> 16) + 63 + static_cast(upperbit) - lz + 1023); + mantissa = (mantissa + (mantissa & 1u)) >> 1u; + const bool carry = mantissa >= (std::uint64_t{2} << 52u); + mantissa = carry ? (std::uint64_t{1} << 52u) : mantissa; + power2 += carry ? 1u : 0u; + return (mantissa & ~(std::uint64_t{1} << 52u)) | (power2 << 52u); +} + +/*! +@brief (binary32 and binary64) the value of a float token of the source with +16 or 17 digits, a point, and no exponent, the shape of most coordinates +written with full precision; whether the token of n has this shape + +These tokens skip the general layout of layout_decimal(): no digits beyond +the 17th, no exponent (and Eisel-Lemire directly, as in layout_float()). +*/ +template +NLOHMANN_VIEW_ALWAYS_INLINE bool coordinate_float(const document_data& d, const node& n, FloatType& value) noexcept +{ + const unsigned int_digits = n.extra & 0xFFu; + const unsigned frac_digits = n.extra >> 8u; + const unsigned count = int_digits + frac_digits; + const auto* const first = reinterpret_cast(d.src + n.off); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + const unsigned negative = first[0] == '-' ? 1u : 0u; + if (count - 16u > 1u || frac_digits == 0 || n.len != negative + count + 1 || d.size - n.off < 33) + { + return false; + } + const unsigned char* const digits = first + negative; + const std::uint64_t w = sixteen_digits(digits, int_digits); + // (the 17th digit follows the point) + const std::uint64_t w17 = (w * 10) + static_cast(digits[17] - '0'); + using format = ieee_binary_format::digits>; + native_float_t result{}; + float_from_bits(coordinate_bits(-static_cast(frac_digits), count == 17 ? w17 : w, format{}) | (static_cast(negative) << format::sign_bit()), result); + value = result; + return true; +} + +#endif + /// the value of the float token of a node, as parse() converts it; floats and /// doubles with at most 19 digits are converted from the digit layout template @@ -2718,6 +2867,13 @@ FloatType float_value(const document_data& d, const node& n) template FloatType float_value(const document_data& d, const node& n, std::true_type /*binary32 or binary64*/) { +#if NLOHMANN_VIEW_VECTOR + FloatType value {}; + if (coordinate_float(d, n, value)) + { + return value; + } +#endif const unsigned int_digits = n.extra & 0xFFu; const unsigned frac_digits = n.extra >> 8u; if (NLOHMANN_VIEW_LIKELY(int_digits + frac_digits <= 19)) // (255 marks "many") @@ -7837,7 +7993,9 @@ NLOHMANN_JSON_NAMESPACE_END #include // size_t #include // uint8_t, uint32_t #include // memcpy, memset +#include // less #include // numeric_limits +#include // unique_ptr #include // integral_constant #include // vector @@ -8212,6 +8370,143 @@ struct depth_counter } }; +/*! +@brief where the double x nearest to a token w * 10^q lies, for 10^15 <= w < +10^17 and -17 <= q <= -1 (Eisel-Lemire, then a fixed-point position) + +dump() writes the shortest decimal in the rounding interval of x (the closest +one if there are several, see zmij::to_shortest()). Its digits are those of a +multiple of 10^g of the token's last digit, with g = dec_exp + 1 - q for x's +decimal exponent dec_exp: the multiple of 10^g nearest to x if it lies in the +interval (at most one does), else the multiple of 10^(g - 1) nearest to x +(see token_digits_1() and token_digits_2()). For g <= 0, the token itself is +in the interval. + +The position is exact in a 64-bit fixed-point number with 56 bits after the +point (of the token's last digit): x / 10^q = 2c * 5^-q / 2^t for x = c * 2^e2 +and t = q + 1 - e2, with at most 56 bits after the point here, and |x / 10^q - +w| is at most half the spacing of the doubles, below 12 units. So only the low +64 bits of 2c * 5^-q are needed. c and e2 come from Eisel-Lemire's product +directly (as eisel_lemire() rounds it: no tie is possible for q < -4, and its +second product is added where it can carry into the result). + +@param[out] offset x / 10^q - w + 32, with 56 bits after the point +@param[out] radius half the spacing of the doubles at x, likewise (+ 1 if + the interval is closed) +@return g; 3 if the digits are not decided here (a power of two, whose + interval is not symmetric, or out of range) +*/ +NLOHMANN_VIEW_ALWAYS_INLINE int locate_token(std::uint64_t w, int q, std::uint64_t& offset, std::uint64_t& radius) noexcept +{ + const int lz = count_leading_zeros(w); + const std::uint64_t normalized = w << static_cast(lz); + const auto index = static_cast(2 * (q - pow5_128_smallest_power)); + const uint128_parts product = full_multiplication(normalized, pow5_128()[index]); + std::uint64_t high = product.high; + if (NLOHMANN_VIEW_UNLIKELY((high & 0x1FFu) == 0x1FFu)) + { + // (as eisel_lemire(): the lower bits may carry into the result) + const std::uint64_t second = full_multiplication(normalized, pow5_128()[index + 1]).high; + high += product.low + second < product.low ? 1u : 0u; + } + const auto upperbit = static_cast(high >> 63u); + const std::uint64_t mantissa = high >> (upperbit + 9); + const std::uint64_t c = (mantissa + (mantissa & 1u)) >> 1u; + // e2 as eisel_lemire() computes the exponent (log2(10) ~ 217706 / 2^16) + const int e2 = static_cast((217706 * static_cast(q)) >> 16) + 63 + static_cast(upperbit) - lz - 52; + const int t = q + 1 - e2; + const int g = ((e2 * 315653) >> 20) + 1 - q; // (dec_exp as in zmij::to_shortest(), for a regular x) + static const std::array fives = {{1u, 5u, 25u, 125u, 625u, 3125u, 15625u, 78125u, 390625u, 1953125u, 9765625u, 48828125u, 244140625u, 1220703125u, 6103515625u, 30517578125u, 152587890625u}}; + const std::uint64_t five = fives[static_cast(-q)]; // 5^-q + const unsigned s = static_cast(56 - t) & 63u; + offset = ((2 * c * five) << s) - (w << 56u) + (std::uint64_t{32} << 56u); + radius = (five << s) + ((c & 1u) == 0 ? 1u : 0u); + // (c = 2^52: a power of two; c = 2^53: rounded up to one. With 8 to 16 + // fraction digits, as the batch's tokens have, t is in [15, 40] and g at + // most 2. x lies halfway between two candidates only if its distance from + // the token is a whole unit (g == 2) or a half (g == 1): such tokens are + // left to the full conversion) + const bool decided = c - ((std::uint64_t{1} << 52u) + 1) < (std::uint64_t{1} << 52u) - 1 + && (offset & ((std::uint64_t{1} << 55u) - 1)) != 0; + return decided ? g : 3; +} + +/// all bits set if b, else none +NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t mask_if(bool b) noexcept +{ + return std::uint64_t{0} - static_cast(b); +} + +/// a if b, else c, without a branch (the conditions of token_digits_1/2() are +/// as good as random, and GCC compiled their ?: as branches: one misprediction +/// per float of canada.json on x86-64) +NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t select_if(bool b, std::uint64_t a, std::uint64_t c) noexcept +{ + return c ^ ((a ^ c) & mask_if(b)); +} + +/// the distance of a position (see locate_token()) from the biased point 32 +NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t biased_distance(std::uint64_t position) noexcept +{ + constexpr std::uint64_t center = std::uint64_t{32} << 56u; + // |position - center|, without a branch: negated (complemented, plus 1) if below + const std::uint64_t below = mask_if(position < center); + return ((position - center) ^ below) - below; +} + +/*! +@brief (g == 1) the digits dump() writes for a token, from the position of its +double (see locate_token()): the multiple of 10 nearest to x if it is in the +interval, else w rounded to x (an exact tie between two candidates, like +754570942.33984373, whose double is 754570942.33984375, is excluded by +locate_token()) + +With |x - w| at most the radius, which is below half the spacing of the +candidates, the multiple of 10 nearest to x is w - r10 or the one above it. +*/ +NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t token_digits_1(std::uint64_t w, std::uint64_t offset, std::uint64_t radius, std::uint64_t r10) noexcept +{ + constexpr std::uint64_t one = std::uint64_t{1} << 56u; + const std::uint64_t from = offset + (r10 * one); // x - (w - r10) (+ 32) + const bool up = from > 37 * one; // (nearer to the multiple above) + const bool shorter = biased_distance(from - (mask_if(up) & (10 * one))) < radius; + const std::uint64_t a = w - r10 + (mask_if(up) & 10u); + const std::uint64_t b = w + ((offset + (one / 2)) >> 56u) - 32; // w rounded to x + return select_if(shorter, a, b); +} + +/*! +@brief (g == 2) the digits dump() writes for a token, as token_digits_1(): the +multiple of 100 nearest to x if it is in the interval, else the multiple of 10 +nearest to x +*/ +NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t token_digits_2(std::uint64_t w, std::uint64_t offset, std::uint64_t radius, std::uint64_t r10, std::uint64_t r100) noexcept +{ + constexpr std::uint64_t one = std::uint64_t{1} << 56u; + const std::uint64_t from = offset + (r100 * one); + const bool up = from > 82 * one; + const bool shorter = biased_distance(from - (mask_if(up) & (100 * one))) < radius; + const std::uint64_t a = w - r100 + (mask_if(up) & 100u); + // x - (w - r10), in [-12, 21]: the multiple of 10 nearest to x is that one, + // or one or two above, or one below + const std::uint64_t y = offset + (r10 * one); + const std::uint64_t steps = static_cast(y > 37 * one) + static_cast(y > 47 * one); + const bool down = y < 27 * one; + const std::uint64_t b = w - r10 + (10 * steps) - (mask_if(down) & 10u); + return select_if(shorter, a, b); +} + +/// the four ASCII digits of x < 10^4, the first in the lowest byte (as +/// little-endian memory holds them) +NLOHMANN_VIEW_ALWAYS_INLINE std::uint32_t four_digit_text(std::uint32_t x) noexcept +{ + const std::uint32_t ab = (x * 5243u) >> 19u; // x / 100 + std::uint32_t v = ab | ((x - (ab * 100u)) << 16u); + const std::uint32_t a = ((v * 103u) >> 10u) & 0x000F000Fu; // / 10 per 16-bit lane + v = a | ((v - (a * 10u)) << 8u); + return v + 0x30303030u; +} + /// how the view's dump() writes a value struct dump_style { @@ -8539,6 +8834,7 @@ class view_serializer std::array buffer; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read std::vector heap; small_stack stack(buffer.data(), buffer.size(), heap); + float_batch batch; const char* const src = m_doc.src; const char* const src_end = src + m_doc.size; char* w = m_out.cursor(); @@ -8678,7 +8974,7 @@ class view_serializer else if (enabled(fast_double::value)) { room(64); - w = write_double_at(w, *n); + w = write_double_ahead(w, src, n, batch); } else { @@ -8703,6 +8999,20 @@ class view_serializer case value_t::array: { const bool object = n->kind == node_kind::object; + // (read-only documents, compact output) a pair of floats, + // like the coordinates of GeoJSON, without a frame + if (enabled(!Editable && !Pretty && !SourceNumbers && fast_double::value) && n->next == 3 && n->len == 2 + && n[1].kind == node_kind::number_float && n[2].kind == node_kind::number_float) + { + room(2 * 64); + *w++ = '['; + w = write_double_ahead(w, src, n + 1, batch); + *w++ = ','; + w = write_double_ahead(w, src, n + 2, batch); + *w++ = ']'; + n += 2; // (the element before the next node) + break; + } room(8); if (n->len == 0) { @@ -8812,6 +9122,254 @@ class view_serializer } } + /// (float_batch) the mark of digits whose rounding carried beyond the + /// token's last four (see mark()) + static constexpr std::uint64_t carried() noexcept + { + return std::uint64_t{1} << 63u; + } + + /// floats of 16 or 17 digits ahead of the walk, converted together (see + /// convert_ahead()), with their digits as mark() keeps them + struct float_batch + { + static constexpr std::size_t capacity() noexcept + { + return 256; + } + /// (on the heap, allocated by the first conversion: a frame of more + /// than a page made every dump of a small document slower) + struct entries + { + std::array at; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::array digits; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + }; + std::unique_ptr storage{}; + const node** at = nullptr; + std::uint64_t* digits = nullptr; + std::size_t count = 0; + std::size_t next = 0; + const node* scanned = nullptr; ///< where the last scan of the node array ended + }; + + /// (doubles) the float at n as dump() writes it, at w (64 bytes of room): + /// from the batch if it holds n + NLOHMANN_VIEW_ALWAYS_INLINE char* write_double_ahead(char* w, const char* src, const node* n, float_batch& batch) + { +#if NLOHMANN_VIEW_VECTOR && NLOHMANN_VIEW_LITTLE_ENDIAN + // (digits 0: not written by the batch; carried(): rounding carried + // beyond the last four digits) + if (batch.next != batch.count && batch.at[batch.next] == n && batch.digits[batch.next] - 1 < carried() - 1) + { + return write_token_digits(w, src, *n, batch.digits[batch.next++]); + } + return write_double_beside(w, n, batch); +#else + static_cast(src); + static_cast(batch); + return write_double_at(w, *n); +#endif + } + +#if NLOHMANN_VIEW_VECTOR && NLOHMANN_VIEW_LITTLE_ENDIAN + /// write_double_ahead() for a float that is not the batch's next one (out + /// of line: the walk is faster with less code in its loop) + NLOHMANN_VIEW_NOINLINE char* write_double_beside(char* w, const node* n, float_batch& batch) + { + // (an editable document's walk can skip nodes of the array) + const std::less before{}; // NOLINT(modernize-use-transparent-functors) C++11 has no std::less<> + while (batch.next != batch.count && before(batch.at[batch.next], n)) + { + ++batch.next; + } + if ((batch.next != batch.count && batch.at[batch.next] == n) || convert_ahead(n, batch)) + { + const std::uint64_t digits = batch.digits[batch.next++]; + if ((digits & carried()) != 0) + { + return write_carried_digits(w, *n, digits & ~carried()); + } + if (digits != 0) + { + return write_token_digits(w, m_doc.src, *n, digits); + } + } + return write_double_at(w, *n); + } + + /*! + @brief convert the floats of 16 or 17 digits among the next nodes of the + array at once; whether n is the first of them + + Such tokens (like the coordinates of GeoJSON, written with 17 digits) are + written as the token with its last four digits replaced + (write_token_digits()). Each step (the digits, locate_token(), + token_digits_1/2()) is a long chain of dependent operations; one loop per + step over many floats lets the processor overlap the floats, which it does + not across the steps of one float (on Apple M1, one loop for the digits and + locate_token() made the dump of canada.json 12 % slower). Each node of the array + is scanned once; a node that the walk does not reach in order (in an + editable document) is written without the batch. + */ + NLOHMANN_VIEW_NOINLINE bool convert_ahead(const node* n, float_batch& batch) + { + const std::less before{}; // NOLINT(modernize-use-transparent-functors) C++11 has no std::less<> + const node* const last = m_doc.nodes + m_doc.node_count; + if (before(n, batch.scanned) || before(n, m_doc.nodes) || !before(n, last)) + { + return false; + } + const char* const src = m_doc.src; + const char* const src_end = src + m_doc.size; + const node* const stop = last - n > 1024 ? n + 1024 : last; + batch.count = 0; + batch.next = 0; + // the digits of each token (and its last four from the text) + std::array digits; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::array exponent; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::array last_four; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + const node* p = n; + if (!batch.storage) + { + batch.storage.reset(new typename float_batch::entries); // NOLINT(cppcoreguidelines-owning-memory,modernize-make-unique): C++11 + batch.at = batch.storage->at.data(); + batch.digits = batch.storage->digits.data(); + } + for (; p != stop && batch.count != float_batch::capacity(); ++p) + { + // a token of the source: 16 or 17 digits, at least 8 after the + // point, no exponent, 32 bytes to read + const unsigned frac_digits = p->extra >> 8u; + const unsigned int_digits = p->extra & 0xFFu; + const unsigned count = int_digits + frac_digits; + if (p->kind == node_kind::number_float && (p->flags & node_flags::storage) == 0 + && count - 16u <= 1u && frac_digits >= 8 && src_end - (src + p->off) >= 33) + { + const auto* const token = reinterpret_cast(src + p->off); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + const unsigned sign = token[0] == '-' ? 1 : 0; + const unsigned char* const first = token + sign; + const std::uint64_t w = sixteen_digits(first, int_digits); + // (the 17th digit follows the point; a token with an exponent + // gets digits outside the range of locate_token(), and is not + // written by the batch. A token 0.xxx with 16 significant + // digits is: its double is written as "0." and its digits too) + const std::uint64_t w17 = (w * 10) + static_cast(first[17] - '0'); + const std::size_t i = batch.count++; + batch.at[i] = p; + if (p->len != sign + count + 1) + { + digits[i] = 0; + } + else + { + digits[i] = count == 17 ? w17 : w; + } + exponent[i] = -static_cast(frac_digits); + // (byte 0: the last digit, byte 1: the last two, bits 16-29: the last four) + std::uint32_t four = 0; + std::memcpy(&four, token + p->len - 4, 4); + four -= 0x30303030u; + const std::uint32_t pairs = ((four & 0x0F0F0F0Fu) * 2561u) >> 8u; // (byte 0, 2: the pairs) + last_four[i] = (four >> 24u) | ((pairs >> 16u & 0xFFu) << 8u) | ((((pairs & 0x00FF00FFu) * 6553601u) >> 16u) << 16u); + } + } + batch.scanned = p; + // the position of each double, and the floats of either kind + std::array offset; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::array radius; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::array kind_1; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::array kind_2; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::size_t count_1 = 0; + std::size_t count_2 = 0; + for (std::size_t i = 0; i != batch.count; ++i) + { + const std::uint64_t w = digits[i]; + // (the range excludes the values of bytes that are not digits, only + // in a damaged image loaded with image_check::bounds) + const int g = w - 1000000000000000u < 99000000000000000u ? locate_token(w, exponent[i], offset[i], radius[i]) : 3; + batch.digits[i] = mask_if(g <= 0) & ((last_four[i] >> 16u) + 1); + kind_1[count_1] = static_cast(i); + count_1 += static_cast(g == 1); + kind_2[count_2] = static_cast(i); + count_2 += static_cast(g == 2); + } + // the digits to write, in one loop per kind (see mark()) + for (std::size_t k = 0; k != count_1; ++k) + { + const std::size_t i = kind_1[k]; + const std::uint64_t s = token_digits_1(digits[i], offset[i], radius[i], last_four[i] & 0xFFu); + batch.digits[i] = mark(digits[i], s, last_four[i] >> 16u); + } + for (std::size_t k = 0; k != count_2; ++k) + { + const std::size_t i = kind_2[k]; + const std::uint64_t s = token_digits_2(digits[i], offset[i], radius[i], last_four[i] & 0xFFu, (last_four[i] >> 8u) & 0xFFu); + batch.digits[i] = mark(digits[i], s, last_four[i] >> 16u); + } + return batch.count != 0 && batch.at[0] == n; + } + + /*! + @brief a float token of 16 or 17 digits as dump() writes it, at w (64 bytes + of room), from the digits of its double (see locate_token()) whose + rounding carried beyond the token's last four digits + + (like 43.42027299999999 -> 43.420273): the digits go to the writer of + Zmij's results as its 15 or 16 leading digits and one more, which omits + the trailing zeros. + */ + NLOHMANN_VIEW_NOINLINE char* write_carried_digits(char* w, const node& n, std::uint64_t digits) const noexcept + { + const int exponent = -(n.extra >> 8u); + const std::uint64_t leading = digits / 10; + const auto last = static_cast(digits - (leading * 10)); + const bool seventeen = digits >= 10000000000000000u; + *w = '-'; + w += m_doc.src[n.off] == '-' ? 1 : 0; + return ::nlohmann::detail::dtoa_impl::write_shortest(w, seventeen ? ::nlohmann::detail::zmij::shortest_decimal{leading, exponent, last, last != 0} + : ::nlohmann::detail::zmij::shortest_decimal{digits, exponent - 1, 0, false}); + } + + /// the digits s of a token with the digits w and the last four digits + /// r4, as the batch keeps them: the new last four digits + 1 (only those + /// are written over the token's), or s marked if it differs from w beyond + /// them (0 stays for a float the batch does not write) + static std::uint64_t mark(std::uint64_t w, std::uint64_t s, std::uint64_t r4) noexcept + { + const std::uint64_t last = s - (w - r4); + return last < 10000 ? last + 1 : s | carried(); + } + + /*! + @brief a float token of 16 or 17 digits as dump() writes it, at w (64 bytes + of room): the token with its last four digits replaced by digits - 1 (see + mark(); the others are the token's), and without trailing zeros after the + point, but one + */ + char* write_token_digits(char* w, const char* src, const node& n, std::uint64_t digits) + { + const char* const token = src + n.off; + std::memcpy(w, token, 32); + char* const end = w + n.len; + const std::uint32_t last = four_digit_text(static_cast(digits - 1)); + std::memcpy(end - 4, &last, 4); + // (the token's text before the new digits, from the source: reading + // the output back would wait for the stores) + std::uint64_t before = 0; + std::memcpy(&before, token + n.len - 12, 8); + const std::uint32_t last_zeros = last ^ 0x30303030u; + const std::uint64_t before_zeros = before ^ 0x3030303030303030u; + if (NLOHMANN_VIEW_UNLIKELY(last_zeros == 0 && before_zeros == 0)) + { + return write_double_at(w, n); // (twelve or more trailing zeros) + } + const unsigned zeros = last_zeros != 0 ? (static_cast(count_leading_zeros(last_zeros)) / 8) - 4 + : 4 + (static_cast(count_leading_zeros(before_zeros)) / 8); + const unsigned frac_digits = n.extra >> 8u; + return end - (zeros < frac_digits ? zeros : frac_digits - 1); + } +#endif + /// a float node as dump() writes it void write_float_node(const node& n) { diff --git a/tests/src/unit-json_view_dump.cpp b/tests/src/unit-json_view_dump.cpp index 6470ab680..be1ec48ad 100644 --- a/tests/src/unit-json_view_dump.cpp +++ b/tests/src/unit-json_view_dump.cpp @@ -35,9 +35,9 @@ using json_view_test::has_duplicate_keys; #include #include -// These tests were split off unit-json_view.cpp, whose object file got too large -// for the MinGW linker (see json_view_test_helpers.hpp). Like that file, this one -// is also built with C++17, as it mentions JSON_HAS_CPP_17. +// These tests were split off unit-json_view.cpp, whose object file got too large for the MinGW +// linker (see json_view_test_helpers.hpp). Like that file, this one is also built +// with C++17, as it mentions JSON_HAS_CPP_17. TEST_CASE("json_view dump") { @@ -150,6 +150,88 @@ TEST_CASE("json_view dump") CHECK(many_doc.root().dump() == json::parse(many_tokens).dump()); } + // tokens of 16 and 17 digits without an exponent (like GeoJSON + // coordinates), which dump() converts in batches: doubles exactly + // halfway between two candidates for the shortest digits, random + // digits, and digits near a shorter decimal (...99999997, + // ...00000003), whose rounding carries; in pairs and among other + // values, so that the batches are interrupted; the last token near the + // end of the text + { + std::mt19937_64 tokens(5627); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed) + const auto draw = [&tokens](std::size_t n) + { + const std::uint64_t r = tokens() % n; + return static_cast(r); + }; + std::string coordinates = "[754570942.33984373,257769445.56835938,858106901.94921875,97944964.876953125," + "[0.5,1e-7],[1.5,-0.0,2.5],[1E2,12.5,0.1,3.25,4.5,5.75,6.125,7.0,8.5],[],[1.5],[1,2.5],[2.5,\"x\"],[[1.5],2.5],"; + for (int i = 0; i < 20000; ++i) + { + const std::size_t length = 16 + draw(2); + std::string digits(1, static_cast('1' + draw(9))); + const std::size_t kept = draw(3) == 0 ? length : 2 + draw(length - 3); + for (std::size_t k = 1; k < kept; ++k) + { + digits += static_cast('0' + draw(10)); + } + digits += std::string(length - kept, draw(2) == 0 ? '9' : '0'); + digits.back() = static_cast('0' + draw(10)); + const std::size_t point = 1 + draw(9); + const std::string token = (draw(2) == 0 ? "-" : "") + digits.substr(0, point) + "." + digits.substr(point); + if (i != 0) + { + coordinates += ','; + } + if (draw(8) == 0) + { + coordinates += R"({"x":)"; + coordinates += token; + coordinates += R"(,"s":"a"})"; + } + else + { + coordinates += '['; + coordinates += token; + coordinates += ','; + coordinates += token; + coordinates += ']'; + } + } + coordinates += ",-65.613616999999977]"; + const json_document parsed = json_document::parse(coordinates); + const ordered_json j = ordered_json::parse(coordinates); + CHECK(parsed.root().dump() == j.dump()); + CHECK(parsed.root().dump(1) == j.dump(1)); + CHECK(parsed.root().dump(-1, ' ', true) == j.dump(-1, ' ', true)); + CHECK(parsed.root().back().get() == j.back().get()); + CHECK(parsed.root().back().dump() == "-65.61361699999998"); + std::size_t i = 0; + for (const json_view v : parsed.root()) + { + const ordered_json& e = j[i++]; + CHECK(v.dump() == e.dump()); + if (v.is_array() && !v.empty() && v[0].is_number()) + { + CHECK(v[0].get() == e[0].get()); + } + } + + // an editable document: edits move some arrays, so that the walk + // leaves the order of the parsed nodes + ordered_json_editable_document editable = ordered_json_editable_document::parse(coordinates); + ordered_json expected = j; + for (std::size_t k = 0; k < 20000; k += 97) + { + if (expected[k].is_array()) + { + editable.push_back(editable.root()[k], ordered_json(1.5)); + expected[k].push_back(1.5); + } + } + CHECK(editable.root().dump() == expected.dump()); + } + // random doubles, written as parse() and dump() would std::mt19937_64 rng(1170); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed) std::string many = "["; @@ -171,673 +253,4 @@ TEST_CASE("json_view dump") const nlohmann::basic_json_document float_document = nlohmann::basic_json_document::parse("[0.1, 1.5e10, 3.4028235e38]"); CHECK(float_document.root().dump() == json_float::parse("[0.1, 1.5e10, 3.4028235e38]").dump()); } - - SECTION("members in document order, all of them") - { - const json_document d = json_document::parse(R"({"b": 1, "a": 2, "b": 3})"); - CHECK(d.root().dump() == R"({"b":1,"a":2,"b":3})"); - CHECK(d.root().dump(1) == "{\n \"b\": 1,\n \"a\": 2,\n \"b\": 3\n}"); - // dump() writes all members, though the lookup finds the first - CHECK(d.root()["b"].dump() == "1"); - } - - SECTION("deep nesting") - { - const std::string deep = std::string(100000, '[') + std::string(100000, ']'); - const json_document deep_document = json_document::parse(deep); - CHECK(deep_document.root().dump() == deep); - } - - SECTION("ensure_ascii: a character that needs an escape at each position of a string") - { - // string lengths around the width of the vector steps, in a string value, a key, and a value near the end of the text - for (const char* const special : - { - "\xC3\xA9", "\xE4\xB8\xAD", "\xF0\x9F\x98\x80", "\x7F", "\\n", "\\u0001", "\\\"", "\\\\" - }) - { - for (std::size_t length = 0; length <= 40; ++length) - { - for (std::size_t position = 0; position <= length; ++position) - { - std::string content(length, 'a'); - content.insert(position, special); - const std::array texts = - { - { - "\"" + content + "\"", - "[\"" + content + "\"]", - "{\"" + content + "\":1}", - } - }; - for (const std::string& text : texts) - { - CAPTURE(text) - const ordered_json_document d = ordered_json_document::parse(text); - const ordered_json j = ordered_json::parse(text); - CHECK(d.root().dump(-1, ' ', true) == j.dump(-1, ' ', true)); - CHECK(d.root().dump(2, ' ', true) == j.dump(2, ' ', true)); - CHECK(d.root().dump() == j.dump()); - } - } - } - } - } - - SECTION("ensure_ascii: every character of the Basic Multilingual Plane and some beyond") - { - // (the two- and three-byte sequences, including the ranges that need a check: E0, ED, EF BF BF) - std::string text = "["; - std::size_t in_chunk = 0; - const auto append = [&](std::uint32_t cp) - { - if (cp < 0x80) - { - if (cp < 0x20 || cp == '"' || cp == '\\') - { - return; - } - text += static_cast(cp); - } - else if (cp < 0x800) - { - text += static_cast(0xC0 | (cp >> 6)); - text += static_cast(0x80 | (cp & 0x3F)); - } - else if (cp < 0x10000) - { - text += static_cast(0xE0 | (cp >> 12)); - text += static_cast(0x80 | ((cp >> 6) & 0x3F)); - text += static_cast(0x80 | (cp & 0x3F)); - } - else - { - text += static_cast(0xF0 | (cp >> 18)); - text += static_cast(0x80 | ((cp >> 12) & 0x3F)); - text += static_cast(0x80 | ((cp >> 6) & 0x3F)); - text += static_cast(0x80 | (cp & 0x3F)); - } - }; - for (std::uint32_t cp = 0x20; cp < 0x10400; ++cp) - { - if (cp >= 0xD800 && cp < 0xE000) - { - continue; - } - if (in_chunk == 0) - { - text += text.size() > 1 ? ",\"" : "\""; - } - append(cp); - if (++in_chunk == 37) - { - text += "\""; - in_chunk = 0; - } - } - text += in_chunk != 0 ? "\"]" : "]"; - const ordered_json_document d = ordered_json_document::parse(text); - const ordered_json j = ordered_json::parse(text); - CHECK(d.root().dump(-1, ' ', true) == j.dump(-1, ' ', true)); - CHECK(d.root().dump(2, ' ', true) == j.dump(2, ' ', true)); - CHECK(d.root().dump() == j.dump()); - CHECK(d.root().dump(2) == j.dump(2)); - } - - SECTION("deep nesting, indented") - { - // indentations of up to and beyond the width of the writer's fixed-size copy - for (const int depth : - { - 1, 5, 20, 70, 300 - }) - { - std::string text; - for (int i = 0; i < depth; ++i) - { - text += i % 2 == 0 ? "[1," : R"({"k":2,"\u00e9":)"; - } - text += "[]"; - for (int i = depth - 1; i >= 0; --i) - { - text += i % 2 == 0 ? "]" : "}"; - } - const ordered_json_document d = ordered_json_document::parse(text); - const ordered_json j = ordered_json::parse(text); - for (const int indent : - { - 0, 1, 2, 3, 8 - }) - { - CHECK(d.root().dump(indent) == j.dump(indent)); - CHECK(d.root().dump(indent, '\t') == j.dump(indent, '\t')); - } - } - } - - SECTION("the output buffer of a small value is small") - { - // an escaped key after the value: its node lies in the arena, so the - // source extent of the value cannot be read from the next node - const std::string big(100000, 'a'); - const std::string text = R"({"small":1,"list":[1,2,3],"k\n":")" + big + R"("})"; - const json_document d = json_document::parse(text); - const auto small = d.root()["small"].dump(); - CHECK(small == "1"); - CHECK(small.capacity() < 4096); - const auto list = d.root()["list"].dump(); - CHECK(list == "[1,2,3]"); - CHECK(list.capacity() < 4096); - CHECK(d.root()["list"].dump(2).capacity() < 4096); - - // the whole document and the large value are unaffected - CHECK(d.root().dump() == ordered_json::parse(text).dump()); - CHECK(d.root()["k\n"].dump() == "\"" + big + "\""); - } - - SECTION("output that outgrows the estimate") - { - // ensure_ascii writes six bytes for each two-byte character - std::string chars; - for (int i = 0; i < 5000; ++i) - { - chars += "\xC3\xA9"; - } - const json_document d = json_document::parse(R"({"a":")" + chars + R"(","k\n":1})"); - const json expected = json::parse("\"" + chars + "\""); - CHECK(d.root()["a"].dump(-1, ' ', true) == expected.dump(-1, ' ', true)); - CHECK(d.root()["a"].dump(-1, ' ', true).size() == 2 + (5000 * 6)); - } - - SECTION("ensure_ascii with strings of the source, decoded strings, and edited strings") - { - // plain ASCII, non-ASCII, escaped (decoded), and mixed strings, as values and as keys; - // the second text has a DEL, which ensure_ascii writes as an escape - for (const char* const del : - {"", "\x7F" - }) - { - const std::string text = std::string("{\"plain\":\"abc\",\"caf\xC3\xA9\":\"\xC3\xA9\xE2\x82\xAC\xF0\x9F\x98\x80\",") - + "\"esc\\n\\\"\":\"a\\tb\\u00e4\\ud83d\\ude00\",\"del\":\"a" + del + "b\",\"\":\"\"," // NOLINT(modernize-raw-string-literal) - + "\"list\":[\"x\",\"\xC3\xBC\",\"\\\\\",\"long ASCII string with more than thirty-two bytes in it\"," - + "\"long string with more than thirty-two bytes and one non-ASCII byte \xC3\xB6\"]," - + "\"n\":[1,2.5,-0,true,null]}"; - const ordered_json j = ordered_json::parse(text); - const ordered_json_document d = ordered_json_document::parse(text); - CHECK(d.root().dump(-1, ' ', true) == j.dump(-1, ' ', true)); - CHECK(d.root().dump(4, ' ', true) == j.dump(4, ' ', true)); - CHECK(d.root().dump() == j.dump()); - CHECK(d.root()["list"].dump(-1, ' ', true) == j["list"].dump(-1, ' ', true)); - CHECK(d.root()["del"].dump(-1, ' ', true) == j["del"].dump(-1, ' ', true)); - - // strings that an edit wrote (the source strings that they replace were plain) - ordered_json_editable_document e = ordered_json_editable_document::parse(text); - ordered_json k = j; - e.set(e.root()["plain"], std::string("\xC3\xA4gypten")); - k["plain"] = "\xC3\xA4gypten"; - e.set(e.root()["list"][0], std::string("ascii after edit")); - k["list"][0] = "ascii after edit"; - e.set(e.root()["list"][1], std::string("\xC3\xA4")); - k["list"][1] = "\xC3\xA4"; - e.set(e.root()["del"], std::string("tab\there")); - k["del"] = "tab\there"; - CHECK(e.root().dump(-1, ' ', true) == k.dump(-1, ' ', true)); - CHECK(e.root().dump(2, ' ', true) == k.dump(2, ' ', true)); - CHECK(e.root().dump() == k.dump()); - - // strings that were edited from or to ASCII text - e.set(e.root()["list"][2], std::string("\xC3\xA4")); - k["list"][2] = "\xC3\xA4"; - e.set(e.root()["list"][4], std::string("plain again")); - k["list"][4] = "plain again"; - CHECK(e.root().dump(-1, ' ', true) == k.dump(-1, ' ', true)); - } - } - - SECTION("ensure_ascii with DEL or a non-ASCII character at every position") - { - // the check whether a string of the source has no byte >= 0x7F reads - // eight bytes at a time and then the rest one by one - for (std::size_t length = 1; length <= 20; ++length) - { - for (std::size_t pos = 0; pos < length; ++pos) - { - for (const char* const c : - { - "\x7F", "\x7E", "\xC3\xA4", "\xC3\xBF" // NOLINT(modernize-raw-string-literal) - }) - { - const std::string s = std::string(pos, 'a') + c + std::string(length - pos - 1, 'b'); - std::string text = "[\""; - text.append(s).append("\",{\"").append(s).append("\":1}]"); - const json_document d = json_document::parse(text); - const json j = json::parse(text); - CHECK(d.root().dump(-1, ' ', true) == j.dump(-1, ' ', true)); - CHECK(d.root().dump(1, ' ', true) == j.dump(1, ' ', true)); - } - } - } - } - - SECTION("streams and discarded views") - { - const json_document d = json_document::parse(R"({"a": [1, 2]})"); - std::ostringstream compact; - compact << d.root(); - CHECK(compact.str() == R"({"a":[1,2]})"); - std::ostringstream pretty; - pretty << std::setw(2) << std::setfill('.') << d.root() << d.root()["a"]; - CHECK(pretty.str() == "{\n..\"a\": [\n....1,\n....2\n..]\n}[1,2]"); - CHECK(json_view().dump() == json(json::value_t::discarded).dump()); - } -} - -TEST_CASE("json_view large objects") -{ - // objects with 128 members or more are looked up with a hash index - for (const std::size_t members : - { - 127u, 128u, 129u, 10000u - }) - { - CAPTURE(members) - std::string text = "{"; - for (std::size_t i = 0; i < members; ++i) - { - text += (i != 0 ? ",\"" : "\"") + std::string(i % 23, 'k') + std::to_string(i) + (i % 7 == 0 ? "\\n" : "") + "\":" + std::to_string(i); - } - text += R"(,"":"empty key","k1":"a duplicate of an earlier key"})"; - const json_document d = json_document::parse(text); - const json_view v = d.root(); - const json j = json::parse(text); - for (std::size_t i = 0; i < members; ++i) - { - const std::string key = std::string(i % 23, 'k') + std::to_string(i) + (i % 7 == 0 ? "\n" : ""); - CHECK(v[key].get() == i); - CHECK(v.contains(key)); - CHECK(v.find(key).key() == key); - CHECK(v.at(key).get() == i); - CHECK(!v.contains(key + "x")); - } - CHECK(v[""].get_string() == "empty key"); - CHECK(v["k1"].get() == 1); // the first of duplicate keys, as for small objects - CHECK(!v.contains("missing")); - CHECK_THROWS_WITH_AS(v.at("missing"), "[json.exception.out_of_range.403] key 'missing' not found", json::out_of_range&); - CHECK(v == j); - CHECK(v.materialize() == j); - } - - SECTION("duplicate keys: the first member wins, with and without a table") - { - // an object of `total` members: the keys "k0".."k" in order, then - // three keys repeated twice more (one copy in the middle, one at the - // end), and two keys repeated once; the value of a member is its - // position, so that the first member of a key can be told apart - struct member - { - std::string key; - std::size_t position; - }; - const auto make_members = [](std::size_t total) - { - std::vector keys; - for (std::size_t i = 0; i + 8 < total; ++i) - { - keys.push_back("k" + std::to_string(i)); - } - const std::size_t n = keys.size(); - const std::array triple = {{3, 17, n - 1}}; - const std::array twice = {{5, n / 2}}; - std::vector ordered = keys; - for (const std::size_t i : triple) - { - ordered.insert(ordered.begin() + static_cast(ordered.size() / 2), keys[i]); - } - for (const std::size_t i : twice) - { - ordered.insert(ordered.begin() + static_cast(ordered.size() / 3), keys[i]); - } - for (const std::size_t i : triple) - { - ordered.push_back(keys[i]); - } - std::vector result; - result.reserve(ordered.size()); - for (std::size_t i = 0; i < ordered.size(); ++i) - { - result.push_back({ordered[i], i}); - } - return result; - }; - - // 100 and 127 members: no table; 128 members and more: a table - for (const std::size_t total : - { - 100u, 127u, 128u, 200u, 5000u - }) - { - CAPTURE(total) - const std::vector members = make_members(total); - REQUIRE(members.size() >= total); - REQUIRE(members.size() >= 100); - std::string text = "{"; - std::map first; - std::map last; - std::map occurrences; - for (const member& m : members) - { - text += (text.size() > 1 ? ",\"" : "\"") + m.key + "\":" + std::to_string(m.position); - first.insert({m.key, m.position}); - last[m.key] = m.position; - ++occurrences[m.key]; - } - text += '}'; - REQUIRE(first.size() < members.size()); - - const json_document d = json_document::parse(text); - const json_view v = d.root(); - const json j = json::parse(text); - CHECK(v.size() == members.size()); // every occurrence is visited - const json m = v.materialize(); - for (const auto& entry : first) - { - CAPTURE(entry.first) - const std::size_t expected = entry.second; - CHECK(v[entry.first].get() == expected); - CHECK(v.at(entry.first).get() == expected); - CHECK(v.find(entry.first).value().get() == expected); - CHECK(v.value(entry.first, std::size_t{0}) == expected); - CHECK(v.contains(entry.first)); - CHECK(v.count(entry.first) == occurrences[entry.first]); - const std::string pointer = "/" + entry.first; - CHECK(v[json::json_pointer(pointer)].get() == expected); - CHECK(v.at(json::json_pointer(pointer)).get() == expected); - CHECK(v.value(json::json_pointer(pointer), std::size_t{0}) == expected); - CHECK(v.contains(json::json_pointer(pointer))); - // materialize() and parse() keep the last value instead - CHECK(j[entry.first].get() == last.at(entry.first)); - CHECK(m[entry.first].get() == last.at(entry.first)); - } - CHECK(!v.contains("k")); - CHECK(v["missing"].is_discarded()); - CHECK(v.materialize() == j); - CHECK(v == j); - } - } - - SECTION("colliding keys") - { - // the hash is not seeded, so keys that all land in one place must not - // make the table build quadratic: such an object gets no table and is - // searched linearly - constexpr std::size_t members = 300; - constexpr std::size_t table_size = 1024; // the table size for 300 members: the next power of two >= 600 - std::vector colliding; - std::vector spread; - for (std::uint64_t counter = 0; colliding.size() < members || spread.size() < members; ++counter) - { - std::string key(8, 'a'); - std::uint64_t x = counter; - for (std::size_t i = 0; i < 8; ++i, x /= 26) - { - key[i] = static_cast('a' + (x % 26)); - } - const bool lands_in_slot_zero = (nlohmann::detail::view::key_hash(key.data(), key.size()) & (table_size - 1)) == 0; - if (lands_in_slot_zero && colliding.size() < members) - { - colliding.push_back(key); - } - else if (!lands_in_slot_zero && spread.size() < members) - { - spread.push_back(key); - } - } - - const auto make_text = [](const std::vector& keys) - { - std::string text = "{"; - for (std::size_t i = 0; i < keys.size(); ++i) - { - text += (i != 0 ? ",\"" : "\"") + keys[i] + "\":" + std::to_string(i % 10); - } - return text + "}"; - }; - const std::string colliding_text = make_text(colliding); - const std::string spread_text = make_text(spread); - - json_document with_collisions = json_document::parse(colliding_text); - json_document without_collisions = json_document::parse(spread_text); - for (const auto* pair : - { - &colliding, &spread - }) - { - const json_view v = (pair == &colliding ? with_collisions : without_collisions).root(); - for (std::size_t i = 0; i < members; ++i) - { - CAPTURE(i) - CHECK(v[(*pair)[i]].get() == i % 10); - CHECK(v.at((*pair)[i]).get() == i % 10); - CHECK(v.find((*pair)[i]).key() == (*pair)[i]); - CHECK(!v.contains((*pair)[i] + "x")); - } - CHECK(!v.contains("missing")); - } - CHECK(with_collisions.root() == json::parse(colliding_text)); - - // only the object with the spread keys got a table (both texts have the - // same length, so the tables are the only difference) - with_collisions.shrink_to_fit(); - without_collisions.shrink_to_fit(); - CHECK(without_collisions.memory_usage() >= with_collisions.memory_usage() + (table_size * sizeof(std::uint32_t))); - } - - SECTION("keys that differ in their last bytes are spread over the table") - { - // The hash must spread the usual families of keys (numbered names, hex - // strings, dates, short names): a key farther than index_max_displacement - // from its home slot would leave the object without a table. The keys - // are placed with linear probing, as the table does. - const auto max_displacement = [](const std::vector& keys) - { - std::size_t slots = 16; - while (slots < 2 * keys.size()) - { - slots *= 2; - } - std::vector used(slots, false); - std::size_t farthest = 0; - for (const std::string& key : keys) - { - std::size_t i = nlohmann::detail::conditional_static_cast(nlohmann::detail::view::key_hash(key.data(), key.size())) & (slots - 1); - std::size_t distance = 0; - while (used[i]) - { - i = (i + 1) & (slots - 1); - ++distance; - } - used[i] = true; - farthest = (std::max)(farthest, distance); - } - return farthest; - }; - // the keys of a printf-like pattern with one %d, %0Nd or %0Nx, formatted with a stream - // (a format string that is not a literal is a Flawfinder error) - const auto numbered = [](const char* format, int count, int step) - { - const std::string pattern(format); - const std::size_t percent = pattern.find('%'); - std::size_t conversion = percent + 1; - std::size_t width = 0; - while (pattern[conversion] >= '0' && pattern[conversion] <= '9') - { - width = (width * 10) + static_cast(pattern[conversion] - '0'); - ++conversion; - } - const bool hex = pattern[conversion] == 'x'; - std::vector keys; - for (int i = 0; i < count; ++i) - { - std::ostringstream key; - key << pattern.substr(0, percent) << std::setfill('0') << std::setw(static_cast(width)) - << (hex ? std::hex : std::dec) << (i * step) << pattern.substr(conversion + 1); - keys.push_back(key.str()); - } - return keys; - }; - for (const char* format : - {"key%d", "%d", "item_%04d", "a%05d", "%03dxyz", "id%d", "user_%d", "%d_suffix", "0x%04x", "A%07d", "12345678%d", "1234567%d", "123456789%d", - "configuration_parameter_value_%04d_extra", "2026-01-%02dT00:00:00Z" - }) - { - CAPTURE(format) - CHECK(max_displacement(numbered(format, std::string(format).find("%02d") != std::string::npos ? 60 : 1000, 1)) < nlohmann::detail::view::index_max_displacement); // NOLINT(abseil-string-find-str-contains) - CHECK(max_displacement(numbered(format, 600, 7)) < nlohmann::detail::view::index_max_displacement); - } - std::mt19937_64 rng(7); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed) - for (const std::size_t length : - { - std::size_t{1}, std::size_t{2}, std::size_t{3}, std::size_t{5}, std::size_t{7}, std::size_t{8}, std::size_t{9}, std::size_t{12}, std::size_t{16}, std::size_t{17}, std::size_t{33} - }) - { - CAPTURE(length) - std::set distinct; - // (there are 26 and 676 keys of one and two letters) - std::size_t wanted = 700; - if (length == 1) - { - wanted = 26; - } - else if (length == 2) - { - wanted = 500; - } - while (distinct.size() < wanted) - { - std::string key(length, 'a'); - for (char& c : key) - { - c = static_cast('a' + (rng() % 26)); - } - distinct.insert(key); - } - CHECK(max_displacement(std::vector(distinct.begin(), distinct.end())) < nlohmann::detail::view::index_max_displacement); - } - } - - SECTION("keys of every length that differ in one byte") - { - // In a large object, a key is found by comparing its bytes with those of the member the - // table points to: up to 16 bytes as two overlapping words, longer ones with memcmp. - // Keys of every length from 0 to 40, and keys that differ from them in one byte. - std::mt19937_64 rng(2026); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed) - std::vector keys; - std::set distinct; - for (std::size_t length = 0; length <= 40; ++length) - { - for (int copy = 0; copy < 5; ++copy) - { - std::string key(length, 'a'); - for (char& c : key) - { - c = static_cast('a' + (rng() % 26)); - } - if (distinct.insert(key).second) - { - keys.push_back(key); - } - } - } - std::string text = "{"; - for (std::size_t i = 0; i < keys.size(); ++i) - { - text += (i != 0 ? ",\"" : "\"") + keys[i] + "\":" + std::to_string(i); - } - text += "}"; - REQUIRE(keys.size() >= 128); // (a table is built for 128 members) - const json_document d = json_document::parse(text); - const json_view v = d.root(); - for (std::size_t i = 0; i < keys.size(); ++i) - { - CAPTURE(keys[i]) - CHECK(v[keys[i]].get() == i); - CHECK(v.at(keys[i]).get() == i); - CHECK(v.contains(keys[i])); - for (std::size_t at = 0; at < keys[i].size(); ++at) - { - std::string other = keys[i]; - other[at] = other[at] == 'z' ? 'y' : 'z'; - CAPTURE(other) - CHECK(v.contains(other) == (distinct.count(other) != 0)); - } - CHECK(v.contains(keys[i] + "a") == (distinct.count(keys[i] + "a") != 0)); - if (!keys[i].empty()) - { - CHECK(v.contains(keys[i].substr(1)) == (distinct.count(keys[i].substr(1)) != 0)); - } - } - } - - SECTION("shrink_to_fit releases the tables' spare capacity") - { - const auto make_text = [](int objects, int members) - { - std::string text = "["; - for (int object = 0; object < objects; ++object) - { - text += object != 0 ? ",{" : "{"; - for (int i = 0, n = members + object; i < n; ++i) - { - text += (i != 0 ? ",\"" : "\"") + std::to_string(i) + "\":" + std::to_string(i); - } - text += '}'; - } - return text + "]"; - }; - const std::string small_text = make_text(5, 150); - const std::string big_text = make_text(40, 400); - - // reading a big text, and then a small one, leaves the spare capacity - // of the big one: shrink_to_fit() brings the document to the size of - // one parsed from the small text alone - json_document d = json_document::parse(big_text); - const std::size_t big = d.memory_usage(); - d.read(small_text); - CHECK(d.memory_usage() >= big); - d.shrink_to_fit(); - json_document fresh = json_document::parse(small_text); - fresh.shrink_to_fit(); - CHECK(d.memory_usage() == fresh.memory_usage()); - CHECK(d.memory_usage() < big / 2); - CHECK(d.root() == json::parse(small_text)); - for (int object = 0; object < 5; ++object) - { - const json_view v = d.root()[static_cast(object)]; - for (int i = 0, n = 150 + object; i < n; ++i) - { - CHECK(v[std::to_string(i)].get() == i); - } - } - } - - SECTION("nested, reused, and in arrays") - { - std::string inner = "{"; - for (int i = 0; i < 300; ++i) - { - inner += (i != 0 ? ",\"m" : "\"m") + std::to_string(i) + "\":" + std::to_string(i); - } - inner += '}'; - const std::string text = "[" + inner + ",{\"x\":" + inner + "}," + inner + "]"; - json_document d = json_document::parse(text); - CHECK(d.root()[0]["m299"].get() == 299); - CHECK(d.root()[1]["x"]["m150"].get() == 150); - CHECK(d.root()[2]["m0"].get() == 0); - const std::size_t with_index = d.memory_usage(); - d.read(std::string("{\"small\": 1}")); - CHECK(d.root()["small"].get() == 1); - d.read(text); - CHECK(d.root()[2]["m7"].get() == 7); - CHECK(d.memory_usage() >= with_index / 2); - } } diff --git a/tests/src/unit-json_view_dump_escape.cpp b/tests/src/unit-json_view_dump_escape.cpp new file mode 100644 index 000000000..3def62df6 --- /dev/null +++ b/tests/src/unit-json_view_dump_escape.cpp @@ -0,0 +1,298 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#include "doctest_compatibility.h" + +#include +using nlohmann::json; +using nlohmann::ordered_json; +using nlohmann::json_document; +using nlohmann::ordered_json_editable_document; +using nlohmann::json_view; +using nlohmann::ordered_json_document; +using nlohmann::ordered_json_view; + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// These tests were split off unit-json_view_dump.cpp, whose object file got too large for the MinGW +// linker (see json_view_test_helpers.hpp). Like that file, this one is also built +// with C++17, as it mentions JSON_HAS_CPP_17. + +TEST_CASE("json_view dump: escaping, nesting, and output buffers") +{ + SECTION("members in document order, all of them") + { + const json_document d = json_document::parse(R"({"b": 1, "a": 2, "b": 3})"); + CHECK(d.root().dump() == R"({"b":1,"a":2,"b":3})"); + CHECK(d.root().dump(1) == "{\n \"b\": 1,\n \"a\": 2,\n \"b\": 3\n}"); + // dump() writes all members, though the lookup finds the first + CHECK(d.root()["b"].dump() == "1"); + } + + SECTION("deep nesting") + { + const std::string deep = std::string(100000, '[') + std::string(100000, ']'); + const json_document deep_document = json_document::parse(deep); + CHECK(deep_document.root().dump() == deep); + } + + SECTION("ensure_ascii: a character that needs an escape at each position of a string") + { + // string lengths around the width of the vector steps, in a string value, a key, and a value near the end of the text + for (const char* const special : + { + "\xC3\xA9", "\xE4\xB8\xAD", "\xF0\x9F\x98\x80", "\x7F", "\\n", "\\u0001", "\\\"", "\\\\" + }) + { + for (std::size_t length = 0; length <= 40; ++length) + { + for (std::size_t position = 0; position <= length; ++position) + { + std::string content(length, 'a'); + content.insert(position, special); + const std::array texts = + { + { + "\"" + content + "\"", + "[\"" + content + "\"]", + "{\"" + content + "\":1}", + } + }; + for (const std::string& text : texts) + { + CAPTURE(text) + const ordered_json_document d = ordered_json_document::parse(text); + const ordered_json j = ordered_json::parse(text); + CHECK(d.root().dump(-1, ' ', true) == j.dump(-1, ' ', true)); + CHECK(d.root().dump(2, ' ', true) == j.dump(2, ' ', true)); + CHECK(d.root().dump() == j.dump()); + } + } + } + } + } + + SECTION("ensure_ascii: every character of the Basic Multilingual Plane and some beyond") + { + // (the two- and three-byte sequences, including the ranges that need a check: E0, ED, EF BF BF) + std::string text = "["; + std::size_t in_chunk = 0; + const auto append = [&](std::uint32_t cp) + { + if (cp < 0x80) + { + if (cp < 0x20 || cp == '"' || cp == '\\') + { + return; + } + text += static_cast(cp); + } + else if (cp < 0x800) + { + text += static_cast(0xC0 | (cp >> 6)); + text += static_cast(0x80 | (cp & 0x3F)); + } + else if (cp < 0x10000) + { + text += static_cast(0xE0 | (cp >> 12)); + text += static_cast(0x80 | ((cp >> 6) & 0x3F)); + text += static_cast(0x80 | (cp & 0x3F)); + } + else + { + text += static_cast(0xF0 | (cp >> 18)); + text += static_cast(0x80 | ((cp >> 12) & 0x3F)); + text += static_cast(0x80 | ((cp >> 6) & 0x3F)); + text += static_cast(0x80 | (cp & 0x3F)); + } + }; + for (std::uint32_t cp = 0x20; cp < 0x10400; ++cp) + { + if (cp >= 0xD800 && cp < 0xE000) + { + continue; + } + if (in_chunk == 0) + { + text += text.size() > 1 ? ",\"" : "\""; + } + append(cp); + if (++in_chunk == 37) + { + text += "\""; + in_chunk = 0; + } + } + text += in_chunk != 0 ? "\"]" : "]"; + const ordered_json_document d = ordered_json_document::parse(text); + const ordered_json j = ordered_json::parse(text); + CHECK(d.root().dump(-1, ' ', true) == j.dump(-1, ' ', true)); + CHECK(d.root().dump(2, ' ', true) == j.dump(2, ' ', true)); + CHECK(d.root().dump() == j.dump()); + CHECK(d.root().dump(2) == j.dump(2)); + } + + SECTION("deep nesting, indented") + { + // indentations of up to and beyond the width of the writer's fixed-size copy + for (const int depth : + { + 1, 5, 20, 70, 300 + }) + { + std::string text; + for (int i = 0; i < depth; ++i) + { + text += i % 2 == 0 ? "[1," : R"({"k":2,"\u00e9":)"; + } + text += "[]"; + for (int i = depth - 1; i >= 0; --i) + { + text += i % 2 == 0 ? "]" : "}"; + } + const ordered_json_document d = ordered_json_document::parse(text); + const ordered_json j = ordered_json::parse(text); + for (const int indent : + { + 0, 1, 2, 3, 8 + }) + { + CHECK(d.root().dump(indent) == j.dump(indent)); + CHECK(d.root().dump(indent, '\t') == j.dump(indent, '\t')); + } + } + } + + SECTION("the output buffer of a small value is small") + { + // an escaped key after the value: its node lies in the arena, so the + // source extent of the value cannot be read from the next node + const std::string big(100000, 'a'); + const std::string text = R"({"small":1,"list":[1,2,3],"k\n":")" + big + R"("})"; + const json_document d = json_document::parse(text); + const auto small = d.root()["small"].dump(); + CHECK(small == "1"); + CHECK(small.capacity() < 4096); + const auto list = d.root()["list"].dump(); + CHECK(list == "[1,2,3]"); + CHECK(list.capacity() < 4096); + CHECK(d.root()["list"].dump(2).capacity() < 4096); + + // the whole document and the large value are unaffected + CHECK(d.root().dump() == ordered_json::parse(text).dump()); + CHECK(d.root()["k\n"].dump() == "\"" + big + "\""); + } + + SECTION("output that outgrows the estimate") + { + // ensure_ascii writes six bytes for each two-byte character + std::string chars; + for (int i = 0; i < 5000; ++i) + { + chars += "\xC3\xA9"; + } + const json_document d = json_document::parse(R"({"a":")" + chars + R"(","k\n":1})"); + const json expected = json::parse("\"" + chars + "\""); + CHECK(d.root()["a"].dump(-1, ' ', true) == expected.dump(-1, ' ', true)); + CHECK(d.root()["a"].dump(-1, ' ', true).size() == 2 + (5000 * 6)); + } + + SECTION("ensure_ascii with strings of the source, decoded strings, and edited strings") + { + // plain ASCII, non-ASCII, escaped (decoded), and mixed strings, as values and as keys; + // the second text has a DEL, which ensure_ascii writes as an escape + for (const char* const del : + {"", "\x7F" + }) + { + const std::string text = std::string("{\"plain\":\"abc\",\"caf\xC3\xA9\":\"\xC3\xA9\xE2\x82\xAC\xF0\x9F\x98\x80\",") + + "\"esc\\n\\\"\":\"a\\tb\\u00e4\\ud83d\\ude00\",\"del\":\"a" + del + "b\",\"\":\"\"," // NOLINT(modernize-raw-string-literal) + + "\"list\":[\"x\",\"\xC3\xBC\",\"\\\\\",\"long ASCII string with more than thirty-two bytes in it\"," + + "\"long string with more than thirty-two bytes and one non-ASCII byte \xC3\xB6\"]," + + "\"n\":[1,2.5,-0,true,null]}"; + const ordered_json j = ordered_json::parse(text); + const ordered_json_document d = ordered_json_document::parse(text); + CHECK(d.root().dump(-1, ' ', true) == j.dump(-1, ' ', true)); + CHECK(d.root().dump(4, ' ', true) == j.dump(4, ' ', true)); + CHECK(d.root().dump() == j.dump()); + CHECK(d.root()["list"].dump(-1, ' ', true) == j["list"].dump(-1, ' ', true)); + CHECK(d.root()["del"].dump(-1, ' ', true) == j["del"].dump(-1, ' ', true)); + + // strings that an edit wrote (the source strings that they replace were plain) + ordered_json_editable_document e = ordered_json_editable_document::parse(text); + ordered_json k = j; + e.set(e.root()["plain"], std::string("\xC3\xA4gypten")); + k["plain"] = "\xC3\xA4gypten"; + e.set(e.root()["list"][0], std::string("ascii after edit")); + k["list"][0] = "ascii after edit"; + e.set(e.root()["list"][1], std::string("\xC3\xA4")); + k["list"][1] = "\xC3\xA4"; + e.set(e.root()["del"], std::string("tab\there")); + k["del"] = "tab\there"; + CHECK(e.root().dump(-1, ' ', true) == k.dump(-1, ' ', true)); + CHECK(e.root().dump(2, ' ', true) == k.dump(2, ' ', true)); + CHECK(e.root().dump() == k.dump()); + + // strings that were edited from or to ASCII text + e.set(e.root()["list"][2], std::string("\xC3\xA4")); + k["list"][2] = "\xC3\xA4"; + e.set(e.root()["list"][4], std::string("plain again")); + k["list"][4] = "plain again"; + CHECK(e.root().dump(-1, ' ', true) == k.dump(-1, ' ', true)); + } + } + + SECTION("ensure_ascii with DEL or a non-ASCII character at every position") + { + // the check whether a string of the source has no byte >= 0x7F reads + // eight bytes at a time and then the rest one by one + for (std::size_t length = 1; length <= 20; ++length) + { + for (std::size_t pos = 0; pos < length; ++pos) + { + for (const char* const c : + { + "\x7F", "\x7E", "\xC3\xA4", "\xC3\xBF" // NOLINT(modernize-raw-string-literal) + }) + { + const std::string s = std::string(pos, 'a') + c + std::string(length - pos - 1, 'b'); + std::string text = "[\""; + text.append(s).append("\",{\"").append(s).append("\":1}]"); + const json_document d = json_document::parse(text); + const json j = json::parse(text); + CHECK(d.root().dump(-1, ' ', true) == j.dump(-1, ' ', true)); + CHECK(d.root().dump(1, ' ', true) == j.dump(1, ' ', true)); + } + } + } + } + + SECTION("streams and discarded views") + { + const json_document d = json_document::parse(R"({"a": [1, 2]})"); + std::ostringstream compact; + compact << d.root(); + CHECK(compact.str() == R"({"a":[1,2]})"); + std::ostringstream pretty; + pretty << std::setw(2) << std::setfill('.') << d.root() << d.root()["a"]; + CHECK(pretty.str() == "{\n..\"a\": [\n....1,\n....2\n..]\n}[1,2]"); + CHECK(json_view().dump() == json(json::value_t::discarded).dump()); + } +} diff --git a/tests/src/unit-json_view_large.cpp b/tests/src/unit-json_view_large.cpp new file mode 100644 index 000000000..89d7db326 --- /dev/null +++ b/tests/src/unit-json_view_large.cpp @@ -0,0 +1,544 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#include "doctest_compatibility.h" + +#include +using nlohmann::json; +using nlohmann::ordered_json; +using nlohmann::json_document; +using nlohmann::ordered_json_editable_document; +using nlohmann::json_view; +using nlohmann::ordered_json_document; +using nlohmann::ordered_json_view; + +#include "json_view_test_helpers.hpp" +using json_view_test::generator; + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// These tests were split off unit-json_view_dump.cpp, whose object file got too large for the MinGW +// linker (see json_view_test_helpers.hpp). Like that file, this one is also built +// with C++17, as it mentions JSON_HAS_CPP_17. + +TEST_CASE("json_view comparison") +{ + SECTION("equality of the values parse() produces") + { + generator g; + std::vector texts; + for (int i = 0; i < 600; ++i) + { + std::string text; + g.value(text, 0); + texts.push_back(text); + // the same value written differently: sorted keys, canonical numbers + texts.push_back(json::parse(text).dump(1)); + } + for (std::size_t i = 0; i + 2 < texts.size(); ++i) + { + for (std::size_t k = i; k < i + 3; ++k) + { + CAPTURE(texts[i]) + CAPTURE(texts[k]) + const json_document a = json_document::parse(texts[i]); + const json_document b = json_document::parse(texts[k]); + const json ja = json::parse(texts[i]); + const json jb = json::parse(texts[k]); + CHECK((a.root() == b.root()) == (ja == jb)); + CHECK((a.root() != b.root()) == (ja != jb)); + CHECK((a.root() == jb) == (ja == jb)); + CHECK((jb == a.root()) == (ja == jb)); + CHECK((a.root() != jb) == (ja != jb)); + CHECK((jb != a.root()) == (ja != jb)); + + // ordered_json compares members in order + const ordered_json_document oa = ordered_json_document::parse(texts[i]); + const ordered_json_document ob = ordered_json_document::parse(texts[k]); + const ordered_json oja = ordered_json::parse(texts[i]); + const ordered_json ojb = ordered_json::parse(texts[k]); + CHECK((oa.root() == ob.root()) == (oja == ojb)); + CHECK((oa.root() == ojb) == (oja == ojb)); + } + } + } + + SECTION("numbers, duplicate keys, member order") + { + const auto same = [](const char* x, const char* y) + { + const json_document dx = json_document::parse(x); + const json_document dy = json_document::parse(y); + return dx.root() == dy.root(); + }; + CHECK(same("1", "1.0")); + CHECK(same("[1, -1, 2.5]", "[1.0, -1.0, 25e-1]")); + CHECK(!same("1", "1.5")); + CHECK(same("18446744073709551615", "18446744073709551615")); + CHECK(same(R"({"a": 1, "a": 2})", R"({"a": 2})")); + CHECK(!same(R"({"a": 1, "a": 2})", R"({"a": 1})")); + CHECK(same(R"({"a": 1, "b": 2})", R"({"b": 2, "a": 1})")); + CHECK(!same(R"({"a": 1})", R"({"a": 1, "b": 2})")); + CHECK(!same("[1, 2]", "[2, 1]")); + CHECK(!same("\"a\"", "\"b\"")); + CHECK(same("\"\\u00e9\"", "\"\xc3\xa9\"")); + CHECK(!same("null", "false")); + CHECK(!same("[]", "{}")); + const ordered_json_document dup = ordered_json_document::parse(R"({"a": 1, "b": 2, "a": 3})"); + const ordered_json_document last = ordered_json_document::parse(R"({"a": 3, "b": 2})"); + CHECK(dup.root() == last.root()); + // (== compares as parse() resolves duplicates, a lookup finds the first) + CHECK(dup.root()["a"].materialize() == 1); + CHECK(dup.root()["a"] != last.root()["a"]); + const ordered_json_document ab = ordered_json_document::parse(R"({"a": 1, "b": 2})"); + const ordered_json_document ba = ordered_json_document::parse(R"({"b": 2, "a": 1})"); + CHECK(ab.root() != ba.root()); + + // discarded values compare as basic_json's do + const json discarded(json::value_t::discarded); + CHECK((json_view() == json_view()) == (discarded == discarded)); // NOLINT(readability-container-size-empty): operator== is tested + CHECK((json_view() == discarded) == (discarded == discarded)); // NOLINT(readability-container-size-empty) + const json_document null_document = json_document::parse("null"); + CHECK(!(json_view() == null_document.root())); // NOLINT(readability-container-size-empty) + CHECK(!(null_document.root() == discarded)); + } + + SECTION("deep nesting") + { + const std::string deep = std::string(100000, '[') + std::string(100000, ']'); + const json_document a = json_document::parse(deep); + const json_document b = json_document::parse(deep); + CHECK(a.root() == b.root()); + CHECK(a.root() == json::parse(deep)); + const std::string other = std::string(100000, '[') + "1" + std::string(100000, ']'); + const json_document c = json_document::parse(other); + CHECK(a.root() != c.root()); + } +} + +TEST_CASE("json_view large objects") +{ + // objects with 128 members or more are looked up with a hash index + for (const std::size_t members : + { + 127u, 128u, 129u, 10000u + }) + { + CAPTURE(members) + std::string text = "{"; + for (std::size_t i = 0; i < members; ++i) + { + text += (i != 0 ? ",\"" : "\"") + std::string(i % 23, 'k') + std::to_string(i) + (i % 7 == 0 ? "\\n" : "") + "\":" + std::to_string(i); + } + text += R"(,"":"empty key","k1":"a duplicate of an earlier key"})"; + const json_document d = json_document::parse(text); + const json_view v = d.root(); + const json j = json::parse(text); + for (std::size_t i = 0; i < members; ++i) + { + const std::string key = std::string(i % 23, 'k') + std::to_string(i) + (i % 7 == 0 ? "\n" : ""); + CHECK(v[key].get() == i); + CHECK(v.contains(key)); + CHECK(v.find(key).key() == key); + CHECK(v.at(key).get() == i); + CHECK(!v.contains(key + "x")); + } + CHECK(v[""].get_string() == "empty key"); + CHECK(v["k1"].get() == 1); // the first of duplicate keys, as for small objects + CHECK(!v.contains("missing")); + CHECK_THROWS_WITH_AS(v.at("missing"), "[json.exception.out_of_range.403] key 'missing' not found", json::out_of_range&); + CHECK(v == j); + CHECK(v.materialize() == j); + } + + SECTION("duplicate keys: the first member wins, with and without a table") + { + // an object of `total` members: the keys "k0".."k" in order, then + // three keys repeated twice more (one copy in the middle, one at the + // end), and two keys repeated once; the value of a member is its + // position, so that the first member of a key can be told apart + struct member + { + std::string key; + std::size_t position; + }; + const auto make_members = [](std::size_t total) + { + std::vector keys; + for (std::size_t i = 0; i + 8 < total; ++i) + { + keys.push_back("k" + std::to_string(i)); + } + const std::size_t n = keys.size(); + const std::array triple = {{3, 17, n - 1}}; + const std::array twice = {{5, n / 2}}; + std::vector ordered = keys; + for (const std::size_t i : triple) + { + ordered.insert(ordered.begin() + static_cast(ordered.size() / 2), keys[i]); + } + for (const std::size_t i : twice) + { + ordered.insert(ordered.begin() + static_cast(ordered.size() / 3), keys[i]); + } + for (const std::size_t i : triple) + { + ordered.push_back(keys[i]); + } + std::vector result; + result.reserve(ordered.size()); + for (std::size_t i = 0; i < ordered.size(); ++i) + { + result.push_back({ordered[i], i}); + } + return result; + }; + + // 100 and 127 members: no table; 128 members and more: a table + for (const std::size_t total : + { + 100u, 127u, 128u, 200u, 5000u + }) + { + CAPTURE(total) + const std::vector members = make_members(total); + REQUIRE(members.size() >= total); + REQUIRE(members.size() >= 100); + std::string text = "{"; + std::map first; + std::map last; + std::map occurrences; + for (const member& m : members) + { + text += (text.size() > 1 ? ",\"" : "\"") + m.key + "\":" + std::to_string(m.position); + first.insert({m.key, m.position}); + last[m.key] = m.position; + ++occurrences[m.key]; + } + text += '}'; + REQUIRE(first.size() < members.size()); + + const json_document d = json_document::parse(text); + const json_view v = d.root(); + const json j = json::parse(text); + CHECK(v.size() == members.size()); // every occurrence is visited + const json m = v.materialize(); + for (const auto& entry : first) + { + CAPTURE(entry.first) + const std::size_t expected = entry.second; + CHECK(v[entry.first].get() == expected); + CHECK(v.at(entry.first).get() == expected); + CHECK(v.find(entry.first).value().get() == expected); + CHECK(v.value(entry.first, std::size_t{0}) == expected); + CHECK(v.contains(entry.first)); + CHECK(v.count(entry.first) == occurrences[entry.first]); + const std::string pointer = "/" + entry.first; + CHECK(v[json::json_pointer(pointer)].get() == expected); + CHECK(v.at(json::json_pointer(pointer)).get() == expected); + CHECK(v.value(json::json_pointer(pointer), std::size_t{0}) == expected); + CHECK(v.contains(json::json_pointer(pointer))); + // materialize() and parse() keep the last value instead + CHECK(j[entry.first].get() == last.at(entry.first)); + CHECK(m[entry.first].get() == last.at(entry.first)); + } + CHECK(!v.contains("k")); + CHECK(v["missing"].is_discarded()); + CHECK(v.materialize() == j); + CHECK(v == j); + } + } + + SECTION("colliding keys") + { + // the hash is not seeded, so keys that all land in one place must not + // make the table build quadratic: such an object gets no table and is + // searched linearly + constexpr std::size_t members = 300; + constexpr std::size_t table_size = 1024; // the table size for 300 members: the next power of two >= 600 + std::vector colliding; + std::vector spread; + for (std::uint64_t counter = 0; colliding.size() < members || spread.size() < members; ++counter) + { + std::string key(8, 'a'); + std::uint64_t x = counter; + for (std::size_t i = 0; i < 8; ++i, x /= 26) + { + key[i] = static_cast('a' + (x % 26)); + } + const bool lands_in_slot_zero = (nlohmann::detail::view::key_hash(key.data(), key.size()) & (table_size - 1)) == 0; + if (lands_in_slot_zero && colliding.size() < members) + { + colliding.push_back(key); + } + else if (!lands_in_slot_zero && spread.size() < members) + { + spread.push_back(key); + } + } + + const auto make_text = [](const std::vector& keys) + { + std::string text = "{"; + for (std::size_t i = 0; i < keys.size(); ++i) + { + text += (i != 0 ? ",\"" : "\"") + keys[i] + "\":" + std::to_string(i % 10); + } + return text + "}"; + }; + const std::string colliding_text = make_text(colliding); + const std::string spread_text = make_text(spread); + + json_document with_collisions = json_document::parse(colliding_text); + json_document without_collisions = json_document::parse(spread_text); + for (const auto* pair : + { + &colliding, &spread + }) + { + const json_view v = (pair == &colliding ? with_collisions : without_collisions).root(); + for (std::size_t i = 0; i < members; ++i) + { + CAPTURE(i) + CHECK(v[(*pair)[i]].get() == i % 10); + CHECK(v.at((*pair)[i]).get() == i % 10); + CHECK(v.find((*pair)[i]).key() == (*pair)[i]); + CHECK(!v.contains((*pair)[i] + "x")); + } + CHECK(!v.contains("missing")); + } + CHECK(with_collisions.root() == json::parse(colliding_text)); + + // only the object with the spread keys got a table (both texts have the + // same length, so the tables are the only difference) + with_collisions.shrink_to_fit(); + without_collisions.shrink_to_fit(); + CHECK(without_collisions.memory_usage() >= with_collisions.memory_usage() + (table_size * sizeof(std::uint32_t))); + } + + SECTION("keys that differ in their last bytes are spread over the table") + { + // The hash must spread the usual families of keys (numbered names, hex + // strings, dates, short names): a key farther than index_max_displacement + // from its home slot would leave the object without a table. The keys + // are placed with linear probing, as the table does. + const auto max_displacement = [](const std::vector& keys) + { + std::size_t slots = 16; + while (slots < 2 * keys.size()) + { + slots *= 2; + } + std::vector used(slots, false); + std::size_t farthest = 0; + for (const std::string& key : keys) + { + std::size_t i = nlohmann::detail::conditional_static_cast(nlohmann::detail::view::key_hash(key.data(), key.size())) & (slots - 1); + std::size_t distance = 0; + while (used[i]) + { + i = (i + 1) & (slots - 1); + ++distance; + } + used[i] = true; + farthest = (std::max)(farthest, distance); + } + return farthest; + }; + // the keys of a printf-like pattern with one %d, %0Nd or %0Nx, formatted with a stream + // (a format string that is not a literal is a Flawfinder error) + const auto numbered = [](const char* format, int count, int step) + { + const std::string pattern(format); + const std::size_t percent = pattern.find('%'); + std::size_t conversion = percent + 1; + std::size_t width = 0; + while (pattern[conversion] >= '0' && pattern[conversion] <= '9') + { + width = (width * 10) + static_cast(pattern[conversion] - '0'); + ++conversion; + } + const bool hex = pattern[conversion] == 'x'; + std::vector keys; + for (int i = 0; i < count; ++i) + { + std::ostringstream key; + key << pattern.substr(0, percent) << std::setfill('0') << std::setw(static_cast(width)) + << (hex ? std::hex : std::dec) << (i * step) << pattern.substr(conversion + 1); + keys.push_back(key.str()); + } + return keys; + }; + for (const char* format : + {"key%d", "%d", "item_%04d", "a%05d", "%03dxyz", "id%d", "user_%d", "%d_suffix", "0x%04x", "A%07d", "12345678%d", "1234567%d", "123456789%d", + "configuration_parameter_value_%04d_extra", "2026-01-%02dT00:00:00Z" + }) + { + CAPTURE(format) + CHECK(max_displacement(numbered(format, std::string(format).find("%02d") != std::string::npos ? 60 : 1000, 1)) < nlohmann::detail::view::index_max_displacement); // NOLINT(abseil-string-find-str-contains) + CHECK(max_displacement(numbered(format, 600, 7)) < nlohmann::detail::view::index_max_displacement); + } + std::mt19937_64 rng(7); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed) + for (const std::size_t length : + { + std::size_t{1}, std::size_t{2}, std::size_t{3}, std::size_t{5}, std::size_t{7}, std::size_t{8}, std::size_t{9}, std::size_t{12}, std::size_t{16}, std::size_t{17}, std::size_t{33} + }) + { + CAPTURE(length) + std::set distinct; + // (there are 26 and 676 keys of one and two letters) + std::size_t wanted = 700; + if (length == 1) + { + wanted = 26; + } + else if (length == 2) + { + wanted = 500; + } + while (distinct.size() < wanted) + { + std::string key(length, 'a'); + for (char& c : key) + { + c = static_cast('a' + (rng() % 26)); + } + distinct.insert(key); + } + CHECK(max_displacement(std::vector(distinct.begin(), distinct.end())) < nlohmann::detail::view::index_max_displacement); + } + } + + SECTION("keys of every length that differ in one byte") + { + // In a large object, a key is found by comparing its bytes with those of the member the + // table points to: up to 16 bytes as two overlapping words, longer ones with memcmp. + // Keys of every length from 0 to 40, and keys that differ from them in one byte. + std::mt19937_64 rng(2026); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed) + std::vector keys; + std::set distinct; + for (std::size_t length = 0; length <= 40; ++length) + { + for (int copy = 0; copy < 5; ++copy) + { + std::string key(length, 'a'); + for (char& c : key) + { + c = static_cast('a' + (rng() % 26)); + } + if (distinct.insert(key).second) + { + keys.push_back(key); + } + } + } + std::string text = "{"; + for (std::size_t i = 0; i < keys.size(); ++i) + { + text += (i != 0 ? ",\"" : "\"") + keys[i] + "\":" + std::to_string(i); + } + text += "}"; + REQUIRE(keys.size() >= 128); // (a table is built for 128 members) + const json_document d = json_document::parse(text); + const json_view v = d.root(); + for (std::size_t i = 0; i < keys.size(); ++i) + { + CAPTURE(keys[i]) + CHECK(v[keys[i]].get() == i); + CHECK(v.at(keys[i]).get() == i); + CHECK(v.contains(keys[i])); + for (std::size_t at = 0; at < keys[i].size(); ++at) + { + std::string other = keys[i]; + other[at] = other[at] == 'z' ? 'y' : 'z'; + CAPTURE(other) + CHECK(v.contains(other) == (distinct.count(other) != 0)); + } + CHECK(v.contains(keys[i] + "a") == (distinct.count(keys[i] + "a") != 0)); + if (!keys[i].empty()) + { + CHECK(v.contains(keys[i].substr(1)) == (distinct.count(keys[i].substr(1)) != 0)); + } + } + } + + SECTION("shrink_to_fit releases the tables' spare capacity") + { + const auto make_text = [](int objects, int members) + { + std::string text = "["; + for (int object = 0; object < objects; ++object) + { + text += object != 0 ? ",{" : "{"; + for (int i = 0, n = members + object; i < n; ++i) + { + text += (i != 0 ? ",\"" : "\"") + std::to_string(i) + "\":" + std::to_string(i); + } + text += '}'; + } + return text + "]"; + }; + const std::string small_text = make_text(5, 150); + const std::string big_text = make_text(40, 400); + + // reading a big text, and then a small one, leaves the spare capacity + // of the big one: shrink_to_fit() brings the document to the size of + // one parsed from the small text alone + json_document d = json_document::parse(big_text); + const std::size_t big = d.memory_usage(); + d.read(small_text); + CHECK(d.memory_usage() >= big); + d.shrink_to_fit(); + json_document fresh = json_document::parse(small_text); + fresh.shrink_to_fit(); + CHECK(d.memory_usage() == fresh.memory_usage()); + CHECK(d.memory_usage() < big / 2); + CHECK(d.root() == json::parse(small_text)); + for (int object = 0; object < 5; ++object) + { + const json_view v = d.root()[static_cast(object)]; + for (int i = 0, n = 150 + object; i < n; ++i) + { + CHECK(v[std::to_string(i)].get() == i); + } + } + } + + SECTION("nested, reused, and in arrays") + { + std::string inner = "{"; + for (int i = 0; i < 300; ++i) + { + inner += (i != 0 ? ",\"m" : "\"m") + std::to_string(i) + "\":" + std::to_string(i); + } + inner += '}'; + const std::string text = "[" + inner + ",{\"x\":" + inner + "}," + inner + "]"; + json_document d = json_document::parse(text); + CHECK(d.root()[0]["m299"].get() == 299); + CHECK(d.root()[1]["x"]["m150"].get() == 150); + CHECK(d.root()[2]["m0"].get() == 0); + const std::size_t with_index = d.memory_usage(); + d.read(std::string("{\"small\": 1}")); + CHECK(d.root()["small"].get() == 1); + d.read(text); + CHECK(d.root()[2]["m7"].get() == 7); + CHECK(d.memory_usage() >= with_index / 2); + } +} From ec6986c918e9ea510dacf5c3f884eb2119610aae Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 1 Nov 2026 17:20:41 +0100 Subject: [PATCH 06/54] Add natvis visualizers for json_document and json_view (#5967) Signed-off-by: Niels Lohmann --- docs/mkdocs/docs/home/debugging.md | 9 +- nlohmann_json.natvis | 11880 ++++++++++++++++ tools/generate_natvis/README.md | 7 +- tools/generate_natvis/generate_natvis.py | 2 +- tools/generate_natvis/nlohmann_json.natvis.j2 | 72 + 5 files changed, 11967 insertions(+), 3 deletions(-) diff --git a/docs/mkdocs/docs/home/debugging.md b/docs/mkdocs/docs/home/debugging.md index d4009dcd0..5722393aa 100644 --- a/docs/mkdocs/docs/home/debugging.md +++ b/docs/mkdocs/docs/home/debugging.md @@ -10,6 +10,13 @@ at its root, a [Natvis](https://learn.microsoft.com/en-us/visualstudio/debugger/ file that gives `json`/`ordered_json` values a friendly, key/value debugger view instead of showing raw internal fields, when debugging with the MSVC debug engine (`cppvsdbg`) in Visual Studio or VS Code. +The file also has visualizers for `json_document` and `json_view` (and their `ordered_json`/editable variants). They +show the value of the document or view, and its elements as an expandable list. Because a document is only a flat index +into the source text, the strings of the direct elements are shown, but the elements of nested arrays and objects are +shown as index nodes (their kind, size, and numbers; strings only by length). Arrays and objects that have been edited +in a `json_editable_document` are not expanded. The visualizers read the internal layout of the MSVC STL (`std::array`, +`std::unique_ptr`). + The checked-in visualizers cover namespace variants with zero, one, or two ABI tags. Configurations with three or more ABI tags are not covered and may show raw internal fields instead. To generate a Natvis file for such a configuration: @@ -29,7 +36,7 @@ ABI tags are not covered and may show raw internal fields instead. To generate a 4. Use the generated file instead of `nlohmann_json.natvis`. -`--abi-tags` can be repeated for several configurations, and `--all` generates every combination (about 60 MB for 12 +`--abi-tags` can be repeated for several configurations, and `--all` generates every combination (about 60 MB for 11 ABI tags, doubling with each further tag). See [`tools/generate_natvis`](https://github.com/nlohmann/json/tree/develop/tools/generate_natvis) for all options. diff --git a/nlohmann_json.natvis b/nlohmann_json.natvis index 1bb733717..33c0caa67 100644 --- a/nlohmann_json.natvis +++ b/nlohmann_json.natvis @@ -79,6 +79,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -153,6 +241,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -227,6 +403,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -301,6 +565,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -375,6 +727,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -449,6 +889,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -523,6 +1051,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -597,6 +1213,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -671,6 +1375,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -745,6 +1537,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -819,6 +1699,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -893,6 +1861,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -967,6 +2023,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -1041,6 +2185,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -1115,6 +2347,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -1189,6 +2509,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -1263,6 +2671,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -1337,6 +2833,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -1411,6 +2995,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -1485,6 +3157,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -1559,6 +3319,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -1633,6 +3481,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -1707,6 +3643,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -1781,6 +3805,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -1855,6 +3967,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -1929,6 +4129,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -2003,6 +4291,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -2077,6 +4453,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -2151,6 +4615,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -2225,6 +4777,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -2299,6 +4939,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -2373,6 +5101,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -2447,6 +5263,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -2521,6 +5425,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -2595,6 +5587,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -2669,6 +5749,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -2743,6 +5911,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -2817,6 +6073,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -2891,6 +6235,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -2965,6 +6397,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -3039,6 +6559,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -3113,6 +6721,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -3187,6 +6883,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -3261,6 +7045,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -3335,6 +7207,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -3409,6 +7369,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -3483,6 +7531,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -3557,6 +7693,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -3631,6 +7855,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -3705,6 +8017,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -3779,6 +8179,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -3853,6 +8341,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -3927,6 +8503,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -4001,6 +8665,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -4075,6 +8827,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -4149,6 +8989,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -4223,6 +9151,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -4297,6 +9313,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -4371,6 +9475,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -4445,6 +9637,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -4519,6 +9799,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -4593,6 +9961,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -4667,6 +10123,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -4741,6 +10285,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -4815,6 +10447,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -4889,6 +10609,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -4963,6 +10771,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -5037,6 +10933,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -5111,6 +11095,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -5185,6 +11257,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -5259,6 +11419,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -5333,6 +11581,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -5407,6 +11743,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -5481,6 +11905,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -5555,6 +12067,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -5629,6 +12229,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -5703,6 +12391,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -5777,6 +12553,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -5851,6 +12715,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -5925,6 +12877,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -5999,6 +13039,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -6073,6 +13201,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -6147,6 +13363,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -6221,6 +13525,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -6295,6 +13687,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -6369,6 +13849,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -6443,6 +14011,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -6517,6 +14173,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -6591,6 +14335,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -6665,6 +14497,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -6739,6 +14659,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -6813,6 +14821,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -6887,6 +14983,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -6961,6 +15145,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -7035,6 +15307,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -7109,6 +15469,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -7183,6 +15631,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -7257,6 +15793,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -7331,6 +15955,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -7405,6 +16117,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -7479,6 +16279,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -7553,6 +16441,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -7627,6 +16603,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -7701,6 +16765,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -7775,6 +16927,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -7849,6 +17089,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -7923,6 +17251,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -7997,6 +17413,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -8071,6 +17575,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -8145,6 +17737,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -8219,6 +17899,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -8293,6 +18061,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -8367,6 +18223,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -8441,6 +18385,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -8515,6 +18547,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -8589,6 +18709,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -8663,6 +18871,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -8737,6 +19033,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -8811,6 +19195,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -8885,6 +19357,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -8959,6 +19519,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -9033,6 +19681,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -9107,6 +19843,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -9181,6 +20005,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -9255,6 +20167,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -9329,6 +20329,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -9403,6 +20491,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -9477,6 +20653,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -9551,6 +20815,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -9625,6 +20977,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -9699,6 +21139,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -9773,6 +21301,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -9847,6 +21463,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -9921,6 +21625,94 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + null @@ -9995,4 +21787,92 @@ + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + + + + + + + + obj ? 2 * n : n + + *p + p = p + 1 + *p + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + + + + + + + + obj ? 2 * n : n + + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_doc->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count + + + + + + + obj ? 2 * n : n + + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + p = p + 1 + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + m_data._Mypair._Myval2->base._Elems[p->flags & 3] + p->off,[p->len]s8 + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + + + diff --git a/tools/generate_natvis/README.md b/tools/generate_natvis/README.md index cc4fc4a8f..712024c57 100644 --- a/tools/generate_natvis/README.md +++ b/tools/generate_natvis/README.md @@ -6,6 +6,11 @@ The checked-in `nlohmann_json.natvis` covers the default namespace (no ABI tags) of ABI tags, with both versioned and unversioned namespace variants. Configurations that enable three or more ABI tags are not covered by this reduced file; use `--abi-tags` or `--all` (see below) to generate a file for them. +The visualizers of `basic_json_document` and `basic_json_view` live in the same inline namespaces as `basic_json`, so the +ABI tags apply to them in the same way, and the `view_mode` template argument is covered by a wildcard: it does not +multiply the namespaces. Each namespace gets three more types (`basic_json_view<*,*>`, `basic_json_document<*,*>`, and +`detail::view::node`), which roughly doubles the size of the file. + The ABI tag list and the library version are parsed from `include/nlohmann/detail/abi_macros.hpp`, so this script must be re-run (via `make natvis`) whenever an `NLOHMANN_JSON_ABI_TAG_*` macro is added to that @@ -35,7 +40,7 @@ version other than the one in `abi_macros.hpp`. |-------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `--list-tags` | Print every ABI tag (in namespace order) with the macro that enables it and the `NLOHMANN_JSON_ABI_TAG_*` identifier, then exit. | | `--abi-tags TAGS` | Add the namespaces of one configuration, given as a comma-separated list of ABI tags (`_diag,_dp,_ns`; the leading underscore is optional, and the order does not matter). Can be repeated for several configurations. Unknown tags are an error. | -| `--all` | Generate the namespaces for every combination of ABI tags. The file size doubles with each tag (about 60 MB for 12 tags). Cannot be combined with `--abi-tags`. | +| `--all` | Generate the namespaces for every combination of ABI tags. The file size doubles with each tag (about 60 MB for 11 tags). Cannot be combined with `--abi-tags`. | | `--output FILE` | Write the Natvis file to `FILE` instead of `nlohmann_json.natvis` in the repository root. | Each configuration gets both the versioned (`json_abi_diag_dp_ns_v3_12_0`) and the unversioned (`json_abi_diag_dp_ns`, diff --git a/tools/generate_natvis/generate_natvis.py b/tools/generate_natvis/generate_natvis.py index 8452d98e0..5a7d1630b 100755 --- a/tools/generate_natvis/generate_natvis.py +++ b/tools/generate_natvis/generate_natvis.py @@ -95,7 +95,7 @@ if __name__ == '__main__': 'optional) of one configuration to add to the default namespaces; can be repeated') selection.add_argument('--all', action='store_true', help='Generate the namespaces for every combination of ABI tags; the file size doubles ' - 'with each ABI tag (about 60 MB for 12 tags)') + 'with each ABI tag (about 60 MB for 11 tags)') parser.add_argument('--list-tags', action='store_true', help='List the ABI tags with the macros that enable them, and exit') parser.add_argument('--output', metavar='FILE', diff --git a/tools/generate_natvis/nlohmann_json.natvis.j2 b/tools/generate_natvis/nlohmann_json.natvis.j2 index bc9e56fd1..e5e7fe03e 100644 --- a/tools/generate_natvis/nlohmann_json.natvis.j2 +++ b/tools/generate_natvis/nlohmann_json.natvis.j2 @@ -5,6 +5,39 @@ +{# The children of the container node `node` (a pointer expression) of the document `doc` (a document_data pointer + expression). A node is only an index into the source text: strings are read from `doc`, so the strings of the + direct children are readable here, but the children of a nested container are shown as nodes (see the node + visualizer below). Containers with edits (flag 8, "moved") are not expanded. The node kinds are the numbers of + node_kind: 1 object, 2 array, 3 string; the float token is shown only by the node visualizer. #} +{% macro children(node, doc=none) %} + + + + + + + obj ? 2 * n : n + +{% if doc %} + {{ doc }}->base._Elems[p->flags & 3] + p->off,[p->len]s8 +{% else %} + *p +{% endif %} + p = p + 1 +{% if doc %} + {{ doc }}->base._Elems[p->flags & 3] + p->off,[p->len]s8 +{% endif %} + *p +{% if doc %} + {{ doc }}->base._Elems[p->flags & 3] + p->off,[p->len]s8 +{% endif %} + *p + p = p + (((int)p->kind == 1 || (int)p->kind == 2) ? p->next : 1) + i = i + 1 + + +{%- endmacro %} {% for ns in namespaces %} @@ -62,5 +95,44 @@ {% endfor %} + + + + null + object, size={len} + array, size={len} + string, length={len} + boolean: {(flags & 4) != 0} + int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next << 32))} + uint64: {(unsigned __int64)len | ((unsigned __int64)next << 32)} + float, length={len} + link + +{{ children('this') }} + + + + + discarded + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8} + {m_doc->base._Elems[m_node->flags & 3] + m_node->off,[m_node->len]s8b} + {*m_node} + +{{ children('m_node', 'm_doc') }} + + + + + discarded + {*m_data._Mypair._Myval2->nodes} + + m_data._Mypair._Myval2->src,[m_data._Mypair._Myval2->size]s8b + m_data._Mypair._Myval2->node_count +{{ children('m_data._Mypair._Myval2->nodes', 'm_data._Mypair._Myval2') }} + + {% endfor %} From 76001c6b4cef42b431ee4f72d50a1b55cb260284 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 1 Nov 2026 17:20:54 +0100 Subject: [PATCH 07/54] BSON reader: find C-string terminators 8 bytes at a time, skip array element names (#5968) Signed-off-by: Niels Lohmann --- .../nlohmann/detail/input/binary_reader.hpp | 40 ++++++----- include/nlohmann/detail/input/string_scan.hpp | 28 ++++++++ single_include/nlohmann/json.hpp | 68 ++++++++++++++----- tests/src/unit-bson.cpp | 37 +++++++++- 4 files changed, 135 insertions(+), 38 deletions(-) diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index 9ed0e4d1b..b671fd84e 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -465,15 +465,17 @@ class binary_reader } const std::size_t element_type_parse_position = chars_read; + // an array's elements are named "0", "1", ... in the wire format, + // and those names are not passed on: they are skipped unless + // error_handler_t::strict must reject ill-formed UTF-8 in them + const bool is_object = container_stack.back().is_object; key.clear(); - if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key))) + if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key, is_object || error_handler == error_handler_t::strict))) { return false; } - // an array's elements are named "0", "1", ... in the wire format, - // and those names are not passed on - if (container_stack.back().is_object && !sax->key(key)) + if (is_object && !sax->key(key)) { return false; } @@ -489,14 +491,18 @@ class binary_reader @brief Parses a C-style string from the BSON input. @param[in,out] result A reference to the string variable where the read string is to be stored. + @param[in] store whether the characters are needed; if not (the names of + an array's elements), they are skipped and @a result may + be left unchanged, and they are not checked against + @ref error_handler @return `true` if the \\x00-byte indicating the end of the string was encountered before the EOF; false` indicates an unexpected EOF. */ - bool get_bson_cstr(string_t& result) + bool get_bson_cstr(string_t& result, const bool store = true) { - if (get_bson_cstr_bulk(result, std::integral_constant {})) + if (get_bson_cstr_bulk(result, store, std::integral_constant {})) { - return check_string_utf8(result, "key"); + return !store || check_string_utf8(result, "key"); } auto out = std::back_inserter(result); @@ -509,7 +515,7 @@ class binary_reader } if (current == 0x00) { - return check_string_utf8(result, "key"); + return !store || check_string_utf8(result, "key"); } *out++ = static_cast(current); } @@ -519,10 +525,11 @@ class binary_reader @brief read a C-style string from contiguous input in one step @param[in,out] result the string to append to + @param[in] store whether to append the characters to @a result @return whether the string was read; if the input has no \\x00-byte, nothing is read, and @ref get_bson_cstr reports the end of the input */ - bool get_bson_cstr_bulk(string_t& result, std::true_type /*bulk*/) + bool get_bson_cstr_bulk(string_t& result, const bool store, std::true_type /*bulk*/) { const std::size_t remaining = ia.bulk_remaining(); if (remaining == 0) @@ -530,18 +537,15 @@ class binary_reader return false; } const auto* const data = reinterpret_cast(ia.bulk_data()); - // a plain loop rather than std::memchr: most keys are short (array - // indices are keys, too), and the call would cost more than it saves - std::size_t length = 0; - while (length < remaining && data[length] != 0x00) - { - ++length; - } + const std::size_t length = find_zero_byte(data, remaining); if (length == remaining) { return false; } - result.append(reinterpret_cast(data), length); + if (store) + { + result.append(reinterpret_cast(data), length); + } // consume the string and its \x00-byte, as the byte-wise path does ia.bulk_skip(length + 1); chars_read += length + 1; @@ -550,7 +554,7 @@ class binary_reader } /// input that is not contiguous: C-style strings are read byte by byte - bool get_bson_cstr_bulk(string_t& /*result*/, std::false_type /*bulk*/) const noexcept + bool get_bson_cstr_bulk(string_t& /*result*/, const bool /*store*/, std::false_type /*bulk*/) const noexcept { return false; } diff --git a/include/nlohmann/detail/input/string_scan.hpp b/include/nlohmann/detail/input/string_scan.hpp index d01a2f1a4..daa9697e1 100644 --- a/include/nlohmann/detail/input/string_scan.hpp +++ b/include/nlohmann/detail/input/string_scan.hpp @@ -90,6 +90,34 @@ JSON_HEDLEY_ALWAYS_INLINE std::size_t find_string_special(const unsigned char* d return n; } +// return the index of the first 0x00 byte in [data, data+n), or n if there is +// none; scans 8 bytes at a time (BSON's C-style strings) +JSON_HEDLEY_ALWAYS_INLINE std::size_t find_zero_byte(const unsigned char* data, std::size_t n) noexcept +{ + constexpr std::uint64_t ones = 0x0101010101010101ull; + constexpr std::uint64_t high = 0x8080808080808080ull; + std::size_t i = 0; + for (; i + 8 <= n; i += 8) + { + const std::uint64_t v = read_eight_bytes(data + i); + const std::uint64_t zero = (v - ones) & ~v & high; + if (zero != 0) + { + // the lowest flagged byte is the first zero byte (see + // find_string_special()) + return i + (static_cast(count_trailing_zeros(zero)) / 8); + } + } + for (; i < n; ++i) + { + if (data[i] == 0x00) + { + return i; + } + } + return n; +} + // classify a byte as one the serializer must NOT copy verbatim when // ensure_ascii is requested: the closing quote, an escape, a control character // (< 0x20), DEL (0x7F), or any non-ASCII byte (>= 0x80). Everything else - diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 83a7090cc..e2ead9c41 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -12458,6 +12458,34 @@ JSON_HEDLEY_ALWAYS_INLINE std::size_t find_string_special(const unsigned char* d return n; } +// return the index of the first 0x00 byte in [data, data+n), or n if there is +// none; scans 8 bytes at a time (BSON's C-style strings) +JSON_HEDLEY_ALWAYS_INLINE std::size_t find_zero_byte(const unsigned char* data, std::size_t n) noexcept +{ + constexpr std::uint64_t ones = 0x0101010101010101ull; + constexpr std::uint64_t high = 0x8080808080808080ull; + std::size_t i = 0; + for (; i + 8 <= n; i += 8) + { + const std::uint64_t v = read_eight_bytes(data + i); + const std::uint64_t zero = (v - ones) & ~v & high; + if (zero != 0) + { + // the lowest flagged byte is the first zero byte (see + // find_string_special()) + return i + (static_cast(count_trailing_zeros(zero)) / 8); + } + } + for (; i < n; ++i) + { + if (data[i] == 0x00) + { + return i; + } + } + return n; +} + // classify a byte as one the serializer must NOT copy verbatim when // ensure_ascii is requested: the closing quote, an escape, a control character // (< 0x20), DEL (0x7F), or any non-ASCII byte (>= 0x80). Everything else - @@ -18241,15 +18269,17 @@ class binary_reader } const std::size_t element_type_parse_position = chars_read; + // an array's elements are named "0", "1", ... in the wire format, + // and those names are not passed on: they are skipped unless + // error_handler_t::strict must reject ill-formed UTF-8 in them + const bool is_object = container_stack.back().is_object; key.clear(); - if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key))) + if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key, is_object || error_handler == error_handler_t::strict))) { return false; } - // an array's elements are named "0", "1", ... in the wire format, - // and those names are not passed on - if (container_stack.back().is_object && !sax->key(key)) + if (is_object && !sax->key(key)) { return false; } @@ -18265,14 +18295,18 @@ class binary_reader @brief Parses a C-style string from the BSON input. @param[in,out] result A reference to the string variable where the read string is to be stored. + @param[in] store whether the characters are needed; if not (the names of + an array's elements), they are skipped and @a result may + be left unchanged, and they are not checked against + @ref error_handler @return `true` if the \\x00-byte indicating the end of the string was encountered before the EOF; false` indicates an unexpected EOF. */ - bool get_bson_cstr(string_t& result) + bool get_bson_cstr(string_t& result, const bool store = true) { - if (get_bson_cstr_bulk(result, std::integral_constant {})) + if (get_bson_cstr_bulk(result, store, std::integral_constant {})) { - return check_string_utf8(result, "key"); + return !store || check_string_utf8(result, "key"); } auto out = std::back_inserter(result); @@ -18285,7 +18319,7 @@ class binary_reader } if (current == 0x00) { - return check_string_utf8(result, "key"); + return !store || check_string_utf8(result, "key"); } *out++ = static_cast(current); } @@ -18295,10 +18329,11 @@ class binary_reader @brief read a C-style string from contiguous input in one step @param[in,out] result the string to append to + @param[in] store whether to append the characters to @a result @return whether the string was read; if the input has no \\x00-byte, nothing is read, and @ref get_bson_cstr reports the end of the input */ - bool get_bson_cstr_bulk(string_t& result, std::true_type /*bulk*/) + bool get_bson_cstr_bulk(string_t& result, const bool store, std::true_type /*bulk*/) { const std::size_t remaining = ia.bulk_remaining(); if (remaining == 0) @@ -18306,18 +18341,15 @@ class binary_reader return false; } const auto* const data = reinterpret_cast(ia.bulk_data()); - // a plain loop rather than std::memchr: most keys are short (array - // indices are keys, too), and the call would cost more than it saves - std::size_t length = 0; - while (length < remaining && data[length] != 0x00) - { - ++length; - } + const std::size_t length = find_zero_byte(data, remaining); if (length == remaining) { return false; } - result.append(reinterpret_cast(data), length); + if (store) + { + result.append(reinterpret_cast(data), length); + } // consume the string and its \x00-byte, as the byte-wise path does ia.bulk_skip(length + 1); chars_read += length + 1; @@ -18326,7 +18358,7 @@ class binary_reader } /// input that is not contiguous: C-style strings are read byte by byte - bool get_bson_cstr_bulk(string_t& /*result*/, std::false_type /*bulk*/) const noexcept + bool get_bson_cstr_bulk(string_t& /*result*/, const bool /*store*/, std::false_type /*bulk*/) const noexcept { return false; } diff --git a/tests/src/unit-bson.cpp b/tests/src/unit-bson.cpp index 8e2d00cbf..f93e0a71b 100644 --- a/tests/src/unit-bson.cpp +++ b/tests/src/unit-bson.cpp @@ -1164,7 +1164,13 @@ TEST_CASE("BSON keys from contiguous and stream input") // contiguous input reads a key up to its \x00-byte in one step, a stream // reads it byte by byte; both must give the same value or error for the // complete document and for every truncation of it - const json j = {{"", true}, {"k", {1, 2, 3}}, {std::string(40, 'x'), {{"nested key", "value"}}}}; + json j = {{"", true}, {"k", {1, 2, 3}}, {std::string(40, 'x'), {{"nested key", "value"}}}}; + // keys and array element names ("0" to "11") that end at every position of + // an 8-byte word, which contiguous input scans at a time + for (std::size_t length = 1; length <= 17; ++length) + { + j[std::string(length, static_cast('a' + length))] = json::array({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, {11}}); + } const std::vector bson = json::to_bson(j); CHECK(json::from_bson(bson) == j); @@ -1184,7 +1190,7 @@ TEST_CASE("BSON keys from contiguous and stream input") } try { - std::istringstream stream(std::string(input.begin(), input.end())); + std::istringstream stream(std::string(reinterpret_cast(input.data()), input.size())); from_stream = json::from_bson(stream).dump(); } catch (const json::parse_error& e) @@ -1194,6 +1200,33 @@ TEST_CASE("BSON keys from contiguous and stream input") CHECK(from_vector == from_stream); } } + +TEST_CASE("BSON array element names") +{ + // an array's element names are not part of the value, so they are not + // passed on; error_handler_t::strict still rejects ill-formed UTF-8 in them + const std::vector bson = + { + 0x1D, 0x00, 0x00, 0x00, // size (little endian) + 0x04, 'a', 0x00, // entry: array "a" + 0x15, 0x00, 0x00, 0x00, // size (little endian) + 0x08, 0xFF, 0x00, 0x01, // entry: boolean named "\xFF", true + 0x08, '1', '2', '3', '4', '5', '6', '7', '8', '9', 0x00, 0x00, // entry: boolean named "123456789", false + 0x00, // end of the array + 0x00 // end of the document + }; + const json expected = {{"a", {true, false}}}; + + CHECK(json::from_bson(bson) == expected); + CHECK(json::from_bson(bson, true, true, json::error_handler_t::replace) == expected); + CHECK(json::from_bson(bson, true, true, json::error_handler_t::ignore) == expected); + + json _; + CHECK_THROWS_WITH_AS(_ = json::from_bson(bson, true, true, json::error_handler_t::strict), "[json.exception.parse_error.113] parse error at byte 14: syntax error while parsing BSON key: invalid string: ill-formed UTF-8 byte", json::parse_error&); + + std::istringstream stream(std::string(reinterpret_cast(bson.data()), bson.size())); + CHECK(json::from_bson(stream) == expected); +} #endif TEST_CASE("Negative size of binary value") From 1a018577cccde18ae47a43195fadccce011257ff Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 1 Nov 2026 17:21:06 +0100 Subject: [PATCH 08/54] json_view: accept() checks the text without writing an index (#5970) Signed-off-by: Niels Lohmann --- .../docs/api/basic_json_document/accept.md | 12 +- include/nlohmann/detail/view/builder.hpp | 138 ++++++++++-- include/nlohmann/detail/view/macro_scope.hpp | 9 + .../nlohmann/detail/view/macro_unscope.hpp | 1 + include/nlohmann/detail/view/simd.hpp | 7 +- include/nlohmann/json_view.hpp | 56 +++-- single_include/nlohmann/json_view.hpp | 211 ++++++++++++++---- tests/src/unit-json_view_builder.cpp | 147 +++++++++++- tests/src/unit-json_view_macros.cpp | 2 +- 9 files changed, 500 insertions(+), 83 deletions(-) diff --git a/docs/mkdocs/docs/api/basic_json_document/accept.md b/docs/mkdocs/docs/api/basic_json_document/accept.md index 1f1070d51..766fd20fa 100644 --- a/docs/mkdocs/docs/api/basic_json_document/accept.md +++ b/docs/mkdocs/docs/api/basic_json_document/accept.md @@ -13,9 +13,15 @@ function never throws an exception for invalid input, and the returned `#!cpp bo is returned. The result is the same as that of `BasicJsonType::accept()` by design; the difference is how the check is made. This -function runs the document's index builder (and discards the index), while `BasicJsonType::accept()` runs the SAX -parser with a handler that ignores every event. Use this one if you go on to parse the same input with -[`parse()`](parse.md) anyway, so that both use the same code; either is fine for a plain yes/no answer. +function runs the document's builder, which checks the text as [`parse()`](parse.md) does, while +`BasicJsonType::accept()` runs the SAX parser with a handler that ignores every event. Use this one if you go on to +parse the same input with [`parse()`](parse.md) anyway, so that both use the same code; either is fine for a plain +yes/no answer. + +When both `ignore_comments` and `ignore_trailing_commas` are `#!cpp false` (the default), the builder writes no index. +With either option set, it builds the index as [`parse()`](parse.md) does and discards it: the result is the same, only +the time differs. (Neither option needs the index; the variant that skips it exists only for the default options to +keep the code size down.) ## Template parameters diff --git a/include/nlohmann/detail/view/builder.hpp b/include/nlohmann/detail/view/builder.hpp index daf0d59bb..4b729ce39 100644 --- a/include/nlohmann/detail/view/builder.hpp +++ b/include/nlohmann/detail/view/builder.hpp @@ -69,7 +69,8 @@ struct parse_failure }; /// FloatType: the number_float_t of the document, whose overflow parse() rejects -template +/// Store: false for accept(): the grammar, number, string and UTF-8 checks run as for parse(), but no node is written +template class builder { public: @@ -245,11 +246,87 @@ class builder return p; } + /// the string from p, the first backslash, control character, ill-formed UTF-8 byte, or the end, checked and not + /// decoded (accept() stores nothing, so the arena is not written): the checks, errors and error positions of + /// slow_string(), for everything up to the closing quote + NLOHMANN_VIEW_NOINLINE decoded check_string(const unsigned char* p) + { + const unsigned char* const end = m_end; + for (;;) + { + if (p == end) + { + return failed(error_code::string_missing_quote, p); + } + const unsigned char c = *p; + if (c == '"') + { + ++p; + return decoded{p, 0, 0}; + } + if (c != '\\') + { + return failed(c < 0x20 ? error_code::string_control_character : error_code::string_utf8, p); + } + ++p; + if (p == end) + { + return failed(error_code::string_missing_quote, p); + } + const unsigned char d = *p++; + if (d == 'u') + { + const int cp = hex4(p); // (moves p past the four digits) + if (NLOHMANN_VIEW_UNLIKELY(cp < 0)) + { + return failed(error_code::string_unicode_hex, p); + } + if (NLOHMANN_VIEW_UNLIKELY((cp & 0xF800) == 0xD800)) // a surrogate + { + if (cp >= 0xDC00) + { + return failed(error_code::string_surrogate_low, p); + } + if (end - p < 2 || p[0] != '\\' || p[1] != 'u') + { + return failed(error_code::string_surrogate_high, p); + } + p += 2; + const int lo = hex4(p); // (moves p past the four digits) + if (lo < 0) + { + return failed(error_code::string_unicode_hex, p); + } + if (lo < 0xDC00 || lo > 0xDFFF) + { + return failed(error_code::string_surrogate_high, p); + } + } + } + else if (!(d == '"' || d == '\\' || d == '/' || d == 'b' || d == 'f' || d == 'n' || d == 'r' || d == 't')) + { + --p; + return failed(error_code::string_escape, p); + } + if (p != end && *p == '\\') + { + continue; // (escapes in a row: no run of text between them) + } + p = scan_string_run(p, end); // the text up to the next backslash, quote, control character, or ill-formed UTF-8 + } + } + /// s points to the start of the string, p to a backslash, a control /// character, an ill-formed UTF-8 byte, or the end: decode the string into /// the arena NLOHMANN_VIEW_NOINLINE decoded slow_string(const unsigned char* s, const unsigned char* p) { + if (!enabled(Store)) + { + // (accept(): the string is checked, not decoded) + static_cast(s); + return check_string(p); + } // single-character escapes; 0: invalid (and 'u', handled separately) static const std::array simple_escape = { @@ -625,10 +702,13 @@ run_found: NLOHMANN_VIEW_ALWAYS_INLINE bool run() { - cold.m_doc.reserve(estimate_nodes(reinterpret_cast(b), static_cast(e - b))); - base = cold.m_doc.nodes; - out = base; - cap = base + cold.m_doc.node_capacity; + if (enabled(Store)) + { + cold.m_doc.reserve(estimate_nodes(reinterpret_cast(b), static_cast(e - b))); + base = cold.m_doc.nodes; + out = base; + cap = base + cold.m_doc.node_capacity; + } if (e - p >= 3 && p[0] == 0xEF && p[1] == 0xBB && p[2] == 0xBF) { @@ -812,7 +892,10 @@ close_object: // an array pays nothing for this) if (NLOHMANN_VIEW_UNLIKELY(cur_count >= document_data::index_min_members)) { - cold.note_large_object(cur_idx); + if (enabled(Store)) + { + cold.note_large_object(cur_idx); + } } close_container: @@ -972,6 +1055,12 @@ indent_done: /// an integer's value); two stores on little-endian targets NLOHMANN_VIEW_ALWAYS_INLINE node* push_node(node_kind k, std::uint8_t flags, std::uint16_t extra, std::size_t off, std::uint64_t second) { + if (!enabled(Store)) + { + // accept(): nothing is stored, so what computes the arguments is not run either. base and out stay + // null: their difference, which open() takes, is 0. + return out; + } if (NLOHMANN_VIEW_UNLIKELY(out == cap)) { const auto n = static_cast(out - base); @@ -1030,9 +1119,12 @@ indent_done: NLOHMANN_VIEW_ALWAYS_INLINE void close() { - node& n = base[cur_idx]; - n.len = cur_count; - n.next = static_cast(out - base) - cur_idx; + if (enabled(Store)) + { + node& n = base[cur_idx]; + n.len = cur_count; + n.next = static_cast(out - base) - cur_idx; + } if (--depth != 0) { if (NLOHMANN_VIEW_LIKELY(depth <= 64)) @@ -1207,34 +1299,44 @@ indent_done: }; /// run the builder with compile-time options -template +template inline bool build_with(document_data& d, const char* src, std::size_t size, bool sentinel, parse_failure& failure) { if (sentinel) { - builder bld(d, src, size); + builder bld(d, src, size); const bool ok = bld.run(); failure = bld.failure(); return ok; } - builder bld(d, src, size); + builder bld(d, src, size); const bool ok = bld.run(); failure = bld.failure(); return ok; } /// sentinel: src[size] is readable and 0 (e.g. std::string); FloatType: the -/// number_float_t of the document +/// number_float_t of the document; validate_only: accept(), only the result is +/// used template -inline bool build(document_data& d, const char* src, std::size_t size, bool comments, bool trailing_commas, bool sentinel, parse_failure& failure) +inline bool build(document_data& d, const char* src, std::size_t size, bool comments, bool trailing_commas, bool sentinel, bool validate_only, + parse_failure& failure) { + if (validate_only && !comments && !trailing_commas) + { + // accept() of plain JSON: nothing is stored. The other option sets + // could skip the stores just as well; they keep the builder that + // stores only so that there are two more instantiations of the parse + // loop, not eight. + return build_with(d, src, size, sentinel, failure); + } if (comments) { - return trailing_commas ? build_with(d, src, size, sentinel, failure) - : build_with(d, src, size, sentinel, failure); + return trailing_commas ? build_with(d, src, size, sentinel, failure) + : build_with(d, src, size, sentinel, failure); } - return trailing_commas ? build_with(d, src, size, sentinel, failure) - : build_with(d, src, size, sentinel, failure); + return trailing_commas ? build_with(d, src, size, sentinel, failure) + : build_with(d, src, size, sentinel, failure); } } // namespace view diff --git a/include/nlohmann/detail/view/macro_scope.hpp b/include/nlohmann/detail/view/macro_scope.hpp index bbe1a240b..f19492648 100644 --- a/include/nlohmann/detail/view/macro_scope.hpp +++ b/include/nlohmann/detail/view/macro_scope.hpp @@ -40,6 +40,15 @@ #define NLOHMANN_VIEW_NOINLINE #endif +// GCC: a function compiled as written, without the clones in which interprocedural +// constant propagation drops an argument that every caller passes as the same constant +// (calling such a clone changes the code GCC generates for the parse loop that calls it) +#if defined(__GNUC__) && !defined(__clang__) + #define NLOHMANN_VIEW_NOCLONE __attribute__((noclone)) +#else + #define NLOHMANN_VIEW_NOCLONE +#endif + #if defined(__GNUC__) || defined(__clang__) #define NLOHMANN_VIEW_NODISCARD __attribute__((warn_unused_result)) #elif defined(_MSC_VER) diff --git a/include/nlohmann/detail/view/macro_unscope.hpp b/include/nlohmann/detail/view/macro_unscope.hpp index a9aec867b..0c4fab4d2 100644 --- a/include/nlohmann/detail/view/macro_unscope.hpp +++ b/include/nlohmann/detail/view/macro_unscope.hpp @@ -15,6 +15,7 @@ #undef NLOHMANN_VIEW_UNLIKELY #undef NLOHMANN_VIEW_ALWAYS_INLINE #undef NLOHMANN_VIEW_NOINLINE +#undef NLOHMANN_VIEW_NOCLONE #undef NLOHMANN_VIEW_NODISCARD #undef NLOHMANN_VIEW_THROW #undef NLOHMANN_VIEW_LITTLE_ENDIAN diff --git a/include/nlohmann/detail/view/simd.hpp b/include/nlohmann/detail/view/simd.hpp index 9dede9b09..f4745f6b8 100644 --- a/include/nlohmann/detail/view/simd.hpp +++ b/include/nlohmann/detail/view/simd.hpp @@ -289,9 +289,12 @@ compares, and the UTF-8 check covers the bytes up to it. Returns where the string scan stops, like scan_string_run: before ill-formed UTF-8 and for the last bytes of the input, the bytes are checked one sequence at a time. Out of line, so that no constants of the check occupy registers in the parse loop. -On x86-64, it is compiled for SSSE3 (see cpu_has_ssse3()). +On x86-64, it is compiled for SSSE3 (see cpu_has_ssse3()). Not cloned: with +the calls of accept()'s builder, GCC 16 would clone it without plain (always +string_plain()), and calling the clone made json_document::parse() run 0.7-1.2 % +more instructions. */ -NLOHMANN_VIEW_SSSE3_TARGET NLOHMANN_VIEW_NOINLINE inline const unsigned char* scan_string_vector(const unsigned char* p, const unsigned char* e, const std::uint8_t* plain) noexcept +NLOHMANN_VIEW_SSSE3_TARGET NLOHMANN_VIEW_NOINLINE NLOHMANN_VIEW_NOCLONE inline const unsigned char* scan_string_vector(const unsigned char* p, const unsigned char* e, const std::uint8_t* plain) noexcept { using lookup = utf8_lookup4<>; const unsigned char* block = p; diff --git a/include/nlohmann/json_view.hpp b/include/nlohmann/json_view.hpp index 9d26d12ea..097e489a4 100644 --- a/include/nlohmann/json_view.hpp +++ b/include/nlohmann/json_view.hpp @@ -1060,7 +1060,7 @@ class basic_json_document const bool ignore_trailing_commas = false) { basic_json_document d; - d.read_range(first, last, allow_exceptions, ignore_comments, ignore_trailing_commas); + d.read_range(first, last, error_mode(allow_exceptions), ignore_comments, ignore_trailing_commas); return d; } @@ -1075,7 +1075,7 @@ class basic_json_document basic_json_document d; std::string buf; append_input(buf, std::forward(input)); - d.build_owned(std::move(buf), allow_exceptions, ignore_comments, ignore_trailing_commas); + d.build_owned(std::move(buf), error_mode(allow_exceptions), ignore_comments, ignore_trailing_commas); return d; } @@ -1088,7 +1088,8 @@ class basic_json_document static bool accept(InputType&& input, const bool ignore_comments = false, const bool ignore_trailing_commas = false) { basic_json_document d; - d.read(std::forward(input), false, ignore_comments, ignore_trailing_commas); + d.read_kind(std::forward(input), on_error::validate, ignore_comments, ignore_trailing_commas, + std::integral_constant::value> {}); return !d.is_discarded(); } @@ -1104,7 +1105,7 @@ class basic_json_document const bool ignore_comments = false, const bool ignore_trailing_commas = false) { - read_kind(std::forward(input), allow_exceptions, ignore_comments, ignore_trailing_commas, + read_kind(std::forward(input), error_mode(allow_exceptions), ignore_comments, ignore_trailing_commas, std::integral_constant::value> {}); } @@ -1463,6 +1464,22 @@ class basic_json_document return idx; } + /// what a failed parse does + enum class on_error : std::uint8_t + { + discard, ///< the document is discarded + raise, ///< parse_error is thrown + validate ///< accept(): the document is discarded, and no node is stored while parsing + }; + + static on_error error_mode(bool allow_exceptions) noexcept + { + return allow_exceptions ? on_error::raise : on_error::discard; + } + + /// NulIsEnd of the builder: a null byte ends the input unless strict_nul_handling + static constexpr bool nul_is_end = !detail::abi_config::strict_nul_handling; + /// create the storage (sized for the input) on first use void ensure_data(const char* src, std::size_t size) { @@ -1474,7 +1491,7 @@ class basic_json_document /// parse a buffer the document takes ownership of (it replaces the /// previous owned copy) - void build_owned(std::string&& buf, bool allow_exceptions, bool comments, bool trailing_commas) + void build_owned(std::string&& buf, on_error allow_exceptions, bool comments, bool trailing_commas) { ensure_data(buf.data(), buf.size()); m_data->owned = std::move(buf); @@ -1484,7 +1501,7 @@ class basic_json_document /// parse a copy of the input; the copy is appended to the cleared owned /// buffer, so that its capacity is reused template - void build_copy(T&& input, bool allow_exceptions, bool comments, bool trailing_commas) + void build_copy(T&& input, on_error allow_exceptions, bool comments, bool trailing_commas) { if (!m_data) { @@ -1505,7 +1522,7 @@ class basic_json_document } /// sentinel: src[size] is readable and 0 (std::string, C strings) - void build(const char* src, std::size_t size, bool allow_exceptions, bool comments, bool trailing_commas, bool owned, bool sentinel) + void build(const char* src, std::size_t size, on_error allow_exceptions, bool comments, bool trailing_commas, bool owned, bool sentinel) { ensure_data(src, size); document_data& d = *m_data; @@ -1532,7 +1549,8 @@ class basic_json_document } else { - ok = detail::view::build < typename BasicJsonType::number_float_t, !detail::abi_config::strict_nul_handling > (d, src, size, comments, trailing_commas, sentinel, failure); + ok = detail::view::build(d, src, size, comments, trailing_commas, sentinel, + allow_exceptions == on_error::validate, failure); } if (NLOHMANN_VIEW_LIKELY(ok)) { @@ -1545,7 +1563,7 @@ class basic_json_document return; } std::vector().swap(d.large_objects); - if (allow_exceptions) + if (allow_exceptions == on_error::raise) { detail::view::throw_parse_failure(failure, src, size, comments, trailing_commas); } @@ -1553,13 +1571,13 @@ class basic_json_document // --- input dispatch (see detail::view::input_kind) --- - void read_kind(std::string&& s, bool ae, bool c, bool tc, std::integral_constant /*unused*/) + void read_kind(std::string&& s, on_error ae, bool c, bool tc, std::integral_constant /*unused*/) { build_owned(std::move(s), ae, c, tc); } template - void read_kind(CharT* s, bool ae, bool c, bool tc, std::integral_constant /*unused*/) + void read_kind(CharT* s, on_error ae, bool c, bool tc, std::integral_constant /*unused*/) { if (s == nullptr) { @@ -1573,7 +1591,7 @@ class basic_json_document } template - void read_kind(Array& a, bool ae, bool c, bool tc, std::integral_constant /*unused*/) + void read_kind(Array& a, on_error ae, bool c, bool tc, std::integral_constant /*unused*/) { using CharT = typename std::remove_cv::type>::type; const std::size_t n = std::extent::value; @@ -1584,7 +1602,7 @@ class basic_json_document } template - void read_kind(const T& s, bool ae, bool c, bool tc, std::integral_constant /*unused*/) + void read_kind(const T& s, on_error ae, bool c, bool tc, std::integral_constant /*unused*/) { // std::basic_string guarantees data()[size()] == 0: use it as sentinel const auto size = static_cast(s.size()); @@ -1593,13 +1611,13 @@ class basic_json_document } template - void read_kind(const T& s, bool ae, bool c, bool tc, std::integral_constant /*unused*/) + void read_kind(const T& s, on_error ae, bool c, bool tc, std::integral_constant /*unused*/) { build_copy(s, ae, c, tc); } template - void read_kind(CharT* s, bool ae, bool c, bool tc, std::integral_constant /*unused*/) + void read_kind(CharT* s, on_error ae, bool c, bool tc, std::integral_constant /*unused*/) { if (s == nullptr) { @@ -1611,13 +1629,13 @@ class basic_json_document } template - void read_kind(T&& input, bool ae, bool c, bool tc, std::integral_constant /*unused*/) + void read_kind(T&& input, on_error ae, bool c, bool tc, std::integral_constant /*unused*/) { build_copy(std::forward(input), ae, c, tc); } template - void read_range(IteratorType first, IteratorType last, bool ae, bool c, bool tc) + void read_range(IteratorType first, IteratorType last, on_error ae, bool c, bool tc) { using value_type = typename std::remove_cv::value_type>::type; // borrow the range where the library's input adapter scans it as one @@ -1628,14 +1646,14 @@ class basic_json_document } template - void read_range_impl(IteratorType first, IteratorType last, bool ae, bool c, bool tc, std::true_type /*contiguous bytes*/) + void read_range_impl(IteratorType first, IteratorType last, on_error ae, bool c, bool tc, std::true_type /*contiguous bytes*/) { const auto size = static_cast(std::distance(first, last)); build(size == 0 ? "" : reinterpret_cast(&*first), size, ae, c, tc, false, size == 0); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) } template - void read_range_impl(IteratorType first, IteratorType last, bool ae, bool c, bool tc, std::false_type /*other*/) + void read_range_impl(IteratorType first, IteratorType last, on_error ae, bool c, bool tc, std::false_type /*other*/) { build_copy(detail::view::make_adapter_input(detail::input_adapter(first, last)), ae, c, tc); } diff --git a/single_include/nlohmann/json_view.hpp b/single_include/nlohmann/json_view.hpp index 7e20544ba..f446b273d 100644 --- a/single_include/nlohmann/json_view.hpp +++ b/single_include/nlohmann/json_view.hpp @@ -157,6 +157,15 @@ #define NLOHMANN_VIEW_NOINLINE #endif +// GCC: a function compiled as written, without the clones in which interprocedural +// constant propagation drops an argument that every caller passes as the same constant +// (calling such a clone changes the code GCC generates for the parse loop that calls it) +#if defined(__GNUC__) && !defined(__clang__) + #define NLOHMANN_VIEW_NOCLONE __attribute__((noclone)) +#else + #define NLOHMANN_VIEW_NOCLONE +#endif + #if defined(__GNUC__) || defined(__clang__) #define NLOHMANN_VIEW_NODISCARD __attribute__((warn_unused_result)) #elif defined(_MSC_VER) @@ -948,9 +957,12 @@ compares, and the UTF-8 check covers the bytes up to it. Returns where the string scan stops, like scan_string_run: before ill-formed UTF-8 and for the last bytes of the input, the bytes are checked one sequence at a time. Out of line, so that no constants of the check occupy registers in the parse loop. -On x86-64, it is compiled for SSSE3 (see cpu_has_ssse3()). +On x86-64, it is compiled for SSSE3 (see cpu_has_ssse3()). Not cloned: with +the calls of accept()'s builder, GCC 16 would clone it without plain (always +string_plain()), and calling the clone made json_document::parse() run 0.7-1.2 % +more instructions. */ -NLOHMANN_VIEW_SSSE3_TARGET NLOHMANN_VIEW_NOINLINE inline const unsigned char* scan_string_vector(const unsigned char* p, const unsigned char* e, const std::uint8_t* plain) noexcept +NLOHMANN_VIEW_SSSE3_TARGET NLOHMANN_VIEW_NOINLINE NLOHMANN_VIEW_NOCLONE inline const unsigned char* scan_string_vector(const unsigned char* p, const unsigned char* e, const std::uint8_t* plain) noexcept { using lookup = utf8_lookup4<>; const unsigned char* block = p; @@ -1311,7 +1323,8 @@ struct parse_failure }; /// FloatType: the number_float_t of the document, whose overflow parse() rejects -template +/// Store: false for accept(): the grammar, number, string and UTF-8 checks run as for parse(), but no node is written +template class builder { public: @@ -1487,11 +1500,87 @@ class builder return p; } + /// the string from p, the first backslash, control character, ill-formed UTF-8 byte, or the end, checked and not + /// decoded (accept() stores nothing, so the arena is not written): the checks, errors and error positions of + /// slow_string(), for everything up to the closing quote + NLOHMANN_VIEW_NOINLINE decoded check_string(const unsigned char* p) + { + const unsigned char* const end = m_end; + for (;;) + { + if (p == end) + { + return failed(error_code::string_missing_quote, p); + } + const unsigned char c = *p; + if (c == '"') + { + ++p; + return decoded{p, 0, 0}; + } + if (c != '\\') + { + return failed(c < 0x20 ? error_code::string_control_character : error_code::string_utf8, p); + } + ++p; + if (p == end) + { + return failed(error_code::string_missing_quote, p); + } + const unsigned char d = *p++; + if (d == 'u') + { + const int cp = hex4(p); // (moves p past the four digits) + if (NLOHMANN_VIEW_UNLIKELY(cp < 0)) + { + return failed(error_code::string_unicode_hex, p); + } + if (NLOHMANN_VIEW_UNLIKELY((cp & 0xF800) == 0xD800)) // a surrogate + { + if (cp >= 0xDC00) + { + return failed(error_code::string_surrogate_low, p); + } + if (end - p < 2 || p[0] != '\\' || p[1] != 'u') + { + return failed(error_code::string_surrogate_high, p); + } + p += 2; + const int lo = hex4(p); // (moves p past the four digits) + if (lo < 0) + { + return failed(error_code::string_unicode_hex, p); + } + if (lo < 0xDC00 || lo > 0xDFFF) + { + return failed(error_code::string_surrogate_high, p); + } + } + } + else if (!(d == '"' || d == '\\' || d == '/' || d == 'b' || d == 'f' || d == 'n' || d == 'r' || d == 't')) + { + --p; + return failed(error_code::string_escape, p); + } + if (p != end && *p == '\\') + { + continue; // (escapes in a row: no run of text between them) + } + p = scan_string_run(p, end); // the text up to the next backslash, quote, control character, or ill-formed UTF-8 + } + } + /// s points to the start of the string, p to a backslash, a control /// character, an ill-formed UTF-8 byte, or the end: decode the string into /// the arena NLOHMANN_VIEW_NOINLINE decoded slow_string(const unsigned char* s, const unsigned char* p) { + if (!enabled(Store)) + { + // (accept(): the string is checked, not decoded) + static_cast(s); + return check_string(p); + } // single-character escapes; 0: invalid (and 'u', handled separately) static const std::array simple_escape = { @@ -1867,10 +1956,13 @@ run_found: NLOHMANN_VIEW_ALWAYS_INLINE bool run() { - cold.m_doc.reserve(estimate_nodes(reinterpret_cast(b), static_cast(e - b))); - base = cold.m_doc.nodes; - out = base; - cap = base + cold.m_doc.node_capacity; + if (enabled(Store)) + { + cold.m_doc.reserve(estimate_nodes(reinterpret_cast(b), static_cast(e - b))); + base = cold.m_doc.nodes; + out = base; + cap = base + cold.m_doc.node_capacity; + } if (e - p >= 3 && p[0] == 0xEF && p[1] == 0xBB && p[2] == 0xBF) { @@ -2054,7 +2146,10 @@ close_object: // an array pays nothing for this) if (NLOHMANN_VIEW_UNLIKELY(cur_count >= document_data::index_min_members)) { - cold.note_large_object(cur_idx); + if (enabled(Store)) + { + cold.note_large_object(cur_idx); + } } close_container: @@ -2214,6 +2309,12 @@ indent_done: /// an integer's value); two stores on little-endian targets NLOHMANN_VIEW_ALWAYS_INLINE node* push_node(node_kind k, std::uint8_t flags, std::uint16_t extra, std::size_t off, std::uint64_t second) { + if (!enabled(Store)) + { + // accept(): nothing is stored, so what computes the arguments is not run either. base and out stay + // null: their difference, which open() takes, is 0. + return out; + } if (NLOHMANN_VIEW_UNLIKELY(out == cap)) { const auto n = static_cast(out - base); @@ -2272,9 +2373,12 @@ indent_done: NLOHMANN_VIEW_ALWAYS_INLINE void close() { - node& n = base[cur_idx]; - n.len = cur_count; - n.next = static_cast(out - base) - cur_idx; + if (enabled(Store)) + { + node& n = base[cur_idx]; + n.len = cur_count; + n.next = static_cast(out - base) - cur_idx; + } if (--depth != 0) { if (NLOHMANN_VIEW_LIKELY(depth <= 64)) @@ -2449,34 +2553,44 @@ indent_done: }; /// run the builder with compile-time options -template +template inline bool build_with(document_data& d, const char* src, std::size_t size, bool sentinel, parse_failure& failure) { if (sentinel) { - builder bld(d, src, size); + builder bld(d, src, size); const bool ok = bld.run(); failure = bld.failure(); return ok; } - builder bld(d, src, size); + builder bld(d, src, size); const bool ok = bld.run(); failure = bld.failure(); return ok; } /// sentinel: src[size] is readable and 0 (e.g. std::string); FloatType: the -/// number_float_t of the document +/// number_float_t of the document; validate_only: accept(), only the result is +/// used template -inline bool build(document_data& d, const char* src, std::size_t size, bool comments, bool trailing_commas, bool sentinel, parse_failure& failure) +inline bool build(document_data& d, const char* src, std::size_t size, bool comments, bool trailing_commas, bool sentinel, bool validate_only, + parse_failure& failure) { + if (validate_only && !comments && !trailing_commas) + { + // accept() of plain JSON: nothing is stored. The other option sets + // could skip the stores just as well; they keep the builder that + // stores only so that there are two more instantiations of the parse + // loop, not eight. + return build_with(d, src, size, sentinel, failure); + } if (comments) { - return trailing_commas ? build_with(d, src, size, sentinel, failure) - : build_with(d, src, size, sentinel, failure); + return trailing_commas ? build_with(d, src, size, sentinel, failure) + : build_with(d, src, size, sentinel, failure); } - return trailing_commas ? build_with(d, src, size, sentinel, failure) - : build_with(d, src, size, sentinel, failure); + return trailing_commas ? build_with(d, src, size, sentinel, failure) + : build_with(d, src, size, sentinel, failure); } } // namespace view @@ -10674,7 +10788,7 @@ class basic_json_document const bool ignore_trailing_commas = false) { basic_json_document d; - d.read_range(first, last, allow_exceptions, ignore_comments, ignore_trailing_commas); + d.read_range(first, last, error_mode(allow_exceptions), ignore_comments, ignore_trailing_commas); return d; } @@ -10689,7 +10803,7 @@ class basic_json_document basic_json_document d; std::string buf; append_input(buf, std::forward(input)); - d.build_owned(std::move(buf), allow_exceptions, ignore_comments, ignore_trailing_commas); + d.build_owned(std::move(buf), error_mode(allow_exceptions), ignore_comments, ignore_trailing_commas); return d; } @@ -10702,7 +10816,8 @@ class basic_json_document static bool accept(InputType&& input, const bool ignore_comments = false, const bool ignore_trailing_commas = false) { basic_json_document d; - d.read(std::forward(input), false, ignore_comments, ignore_trailing_commas); + d.read_kind(std::forward(input), on_error::validate, ignore_comments, ignore_trailing_commas, + std::integral_constant::value> {}); return !d.is_discarded(); } @@ -10718,7 +10833,7 @@ class basic_json_document const bool ignore_comments = false, const bool ignore_trailing_commas = false) { - read_kind(std::forward(input), allow_exceptions, ignore_comments, ignore_trailing_commas, + read_kind(std::forward(input), error_mode(allow_exceptions), ignore_comments, ignore_trailing_commas, std::integral_constant::value> {}); } @@ -11077,6 +11192,22 @@ class basic_json_document return idx; } + /// what a failed parse does + enum class on_error : std::uint8_t + { + discard, ///< the document is discarded + raise, ///< parse_error is thrown + validate ///< accept(): the document is discarded, and no node is stored while parsing + }; + + static on_error error_mode(bool allow_exceptions) noexcept + { + return allow_exceptions ? on_error::raise : on_error::discard; + } + + /// NulIsEnd of the builder: a null byte ends the input unless strict_nul_handling + static constexpr bool nul_is_end = !detail::abi_config::strict_nul_handling; + /// create the storage (sized for the input) on first use void ensure_data(const char* src, std::size_t size) { @@ -11088,7 +11219,7 @@ class basic_json_document /// parse a buffer the document takes ownership of (it replaces the /// previous owned copy) - void build_owned(std::string&& buf, bool allow_exceptions, bool comments, bool trailing_commas) + void build_owned(std::string&& buf, on_error allow_exceptions, bool comments, bool trailing_commas) { ensure_data(buf.data(), buf.size()); m_data->owned = std::move(buf); @@ -11098,7 +11229,7 @@ class basic_json_document /// parse a copy of the input; the copy is appended to the cleared owned /// buffer, so that its capacity is reused template - void build_copy(T&& input, bool allow_exceptions, bool comments, bool trailing_commas) + void build_copy(T&& input, on_error allow_exceptions, bool comments, bool trailing_commas) { if (!m_data) { @@ -11119,7 +11250,7 @@ class basic_json_document } /// sentinel: src[size] is readable and 0 (std::string, C strings) - void build(const char* src, std::size_t size, bool allow_exceptions, bool comments, bool trailing_commas, bool owned, bool sentinel) + void build(const char* src, std::size_t size, on_error allow_exceptions, bool comments, bool trailing_commas, bool owned, bool sentinel) { ensure_data(src, size); document_data& d = *m_data; @@ -11146,7 +11277,8 @@ class basic_json_document } else { - ok = detail::view::build < typename BasicJsonType::number_float_t, !detail::abi_config::strict_nul_handling > (d, src, size, comments, trailing_commas, sentinel, failure); + ok = detail::view::build(d, src, size, comments, trailing_commas, sentinel, + allow_exceptions == on_error::validate, failure); } if (NLOHMANN_VIEW_LIKELY(ok)) { @@ -11159,7 +11291,7 @@ class basic_json_document return; } std::vector().swap(d.large_objects); - if (allow_exceptions) + if (allow_exceptions == on_error::raise) { detail::view::throw_parse_failure(failure, src, size, comments, trailing_commas); } @@ -11167,13 +11299,13 @@ class basic_json_document // --- input dispatch (see detail::view::input_kind) --- - void read_kind(std::string&& s, bool ae, bool c, bool tc, std::integral_constant /*unused*/) + void read_kind(std::string&& s, on_error ae, bool c, bool tc, std::integral_constant /*unused*/) { build_owned(std::move(s), ae, c, tc); } template - void read_kind(CharT* s, bool ae, bool c, bool tc, std::integral_constant /*unused*/) + void read_kind(CharT* s, on_error ae, bool c, bool tc, std::integral_constant /*unused*/) { if (s == nullptr) { @@ -11187,7 +11319,7 @@ class basic_json_document } template - void read_kind(Array& a, bool ae, bool c, bool tc, std::integral_constant /*unused*/) + void read_kind(Array& a, on_error ae, bool c, bool tc, std::integral_constant /*unused*/) { using CharT = typename std::remove_cv::type>::type; const std::size_t n = std::extent::value; @@ -11198,7 +11330,7 @@ class basic_json_document } template - void read_kind(const T& s, bool ae, bool c, bool tc, std::integral_constant /*unused*/) + void read_kind(const T& s, on_error ae, bool c, bool tc, std::integral_constant /*unused*/) { // std::basic_string guarantees data()[size()] == 0: use it as sentinel const auto size = static_cast(s.size()); @@ -11207,13 +11339,13 @@ class basic_json_document } template - void read_kind(const T& s, bool ae, bool c, bool tc, std::integral_constant /*unused*/) + void read_kind(const T& s, on_error ae, bool c, bool tc, std::integral_constant /*unused*/) { build_copy(s, ae, c, tc); } template - void read_kind(CharT* s, bool ae, bool c, bool tc, std::integral_constant /*unused*/) + void read_kind(CharT* s, on_error ae, bool c, bool tc, std::integral_constant /*unused*/) { if (s == nullptr) { @@ -11225,13 +11357,13 @@ class basic_json_document } template - void read_kind(T&& input, bool ae, bool c, bool tc, std::integral_constant /*unused*/) + void read_kind(T&& input, on_error ae, bool c, bool tc, std::integral_constant /*unused*/) { build_copy(std::forward(input), ae, c, tc); } template - void read_range(IteratorType first, IteratorType last, bool ae, bool c, bool tc) + void read_range(IteratorType first, IteratorType last, on_error ae, bool c, bool tc) { using value_type = typename std::remove_cv::value_type>::type; // borrow the range where the library's input adapter scans it as one @@ -11242,14 +11374,14 @@ class basic_json_document } template - void read_range_impl(IteratorType first, IteratorType last, bool ae, bool c, bool tc, std::true_type /*contiguous bytes*/) + void read_range_impl(IteratorType first, IteratorType last, on_error ae, bool c, bool tc, std::true_type /*contiguous bytes*/) { const auto size = static_cast(std::distance(first, last)); build(size == 0 ? "" : reinterpret_cast(&*first), size, ae, c, tc, false, size == 0); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) } template - void read_range_impl(IteratorType first, IteratorType last, bool ae, bool c, bool tc, std::false_type /*other*/) + void read_range_impl(IteratorType first, IteratorType last, on_error ae, bool c, bool tc, std::false_type /*other*/) { build_copy(detail::view::make_adapter_input(detail::input_adapter(first, last)), ae, c, tc); } @@ -11380,6 +11512,7 @@ class tuple_element> // NOLINT(cert #undef NLOHMANN_VIEW_UNLIKELY #undef NLOHMANN_VIEW_ALWAYS_INLINE #undef NLOHMANN_VIEW_NOINLINE +#undef NLOHMANN_VIEW_NOCLONE #undef NLOHMANN_VIEW_NODISCARD #undef NLOHMANN_VIEW_THROW #undef NLOHMANN_VIEW_LITTLE_ENDIAN diff --git a/tests/src/unit-json_view_builder.cpp b/tests/src/unit-json_view_builder.cpp index 7ac7a7d40..418f219da 100644 --- a/tests/src/unit-json_view_builder.cpp +++ b/tests/src/unit-json_view_builder.cpp @@ -13,6 +13,7 @@ #include #include #include + #include // json_document::accept #else #include // the single header contains the internal headers #endif @@ -59,7 +60,8 @@ built build(const std::string& text, bool comments, bool trailing_commas, bool s r.copy.assign(text.begin(), text.end()); src = r.copy.data(); } - r.ok = nlohmann::detail::view::build < FloatType, !nlohmann::detail::abi_config::strict_nul_handling > (*r.data, src, text.size(), comments, trailing_commas, sentinel, r.failure); + constexpr bool nul_is_end = !nlohmann::detail::abi_config::strict_nul_handling; + r.ok = nlohmann::detail::view::build(*r.data, src, text.size(), comments, trailing_commas, sentinel, false, r.failure); r.data->src = src; r.data->base[0] = src; r.data->base[1] = r.data->arena.data(); @@ -172,6 +174,22 @@ void check_string(const std::string& content) } } +// accept() stores no nodes: it must accept and reject what json::accept does, for all options and for inputs with +// and without a terminating NUL +void check_accept(const std::string& text) +{ + CAPTURE(text) + const std::vector bytes(text.begin(), text.end()); + for (int options = 0; options < 4; ++options) + { + const bool comments = (options & 1) != 0; + const bool trailing_commas = (options & 2) != 0; + const bool accepted = json::accept(text, comments, trailing_commas); + CHECK(nlohmann::json_document::accept(text, comments, trailing_commas) == accepted); + CHECK(nlohmann::json_document::accept(bytes, comments, trailing_commas) == accepted); + } +} + // a small deterministic generator of documents struct generator { @@ -423,6 +441,133 @@ TEST_CASE("json_view builder") } } + SECTION("accept() without an index") + { + for (const char* text : + { + "", " ", "\n", "null", "nul", "true", "[", "]", "[]", "{}", "[1,2,3]", "[1,2,3", "[1,2,3,]", "{\"a\":1,}", "{\"a\"}", "{\"a\":}", + "[1] x", "[1][2]", "\xEF\xBB\xBF[1]", "[\"a\\", R"(["\u12"])", R"(["\ud800"])", "[\"\x01\"]", "[\"\xff\"]", "[01]", "[1.]", + "[.5]", "[-]", "[1e]", "// c\n[1]", "/* c */ {\"a\" /* x */ : 1}", "/* c", "[1] // c", "1e400", "-1e400", "123456789012345678901234567890", + R"({"a":1,"a":2})", "[[[]]]", R"("\u00e4\n\ud83d\ude00")" + }) + { + check_accept(text); + } + // a null byte inside the text, and after it + check_accept(std::string("[1,\0]", 5)); + check_accept(std::string("[1]\0", 4)); + check_accept(std::string("\0", 1)); + + // generated documents and damaged copies, as for the builder above + generator g; + for (int i = 0; i < 2000; ++i) + { + std::string text; + g.value(text, 0); + check_accept(text); + std::string damaged = text; + const auto at = static_cast(g.r(static_cast(damaged.size()))); + static const char replacements[] = {'x', '"', '\\', ',', ':', ']', '}', '[', '{', '1', '-', '.', 'e', '\0', '\n', '/'}; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + damaged[at] = replacements[g.r(16)]; + check_accept(damaged); + check_accept(text.substr(0, at)); + } + + // nesting beyond the 64 levels that are kept on the builder's own stack, closed and not closed + for (const std::size_t depth : + { + 63u, 64u, 65u, 66u, 200u, 1000u + }) + { + check_accept(std::string(depth, '[') + std::string(depth, ']')); + check_accept(std::string(depth, '[') + std::string(depth - 1, ']')); + std::string objects; + for (std::size_t i = 0; i < depth; ++i) + { + objects += "{\"k\":"; + } + objects += "1" + std::string(depth, '}'); + check_accept(objects); + check_accept(objects.substr(0, objects.size() - 1)); + } + + // objects with enough members to get a hash index when they are indexed (and duplicate keys) + std::string large = "{"; + for (int i = 0; i < 300; ++i) + { + large += (i == 0 ? "\"k" : ",\"k") + std::to_string(i % 200) + "\":" + std::to_string(i); + } + check_accept(large + "}"); + check_accept(large); + check_accept("[" + large + "},{" + large.substr(1) + "}]"); + + // the overflow check uses the floating-point type of the document + using float_json = nlohmann::basic_json; + for (const char* text : + { + "1e39", "-1e39", "3.4028235e38", "3.4028236e38", "[1.5e38, 3.5e38]", R"({"a": 1e-50, "b": 1e39})", "340282356779733661637539395458142568448" + }) + { + CAPTURE(text) + CHECK(nlohmann::basic_json_document::accept(text) == float_json::accept(text)); + } + + // strings are checked and not decoded: runs of text of every length from 0 to 40 between valid and invalid + // escapes, UTF-8 sequences and control characters, in values and in keys, cut near the end of the text + const std::array valid = {{"\\n", "\\\"", "\\\\", "\\/", "\\t", "\\u00e4", "\\u20ac", "\\ud83d\\ude00", "\\u0000"}}; + const std::array invalid = {{"\\x", "\\ud800", "\\udc00", "\\u12G4", "\\u12", "\\ud83dx", "\\ud83d\\u0041", "\\ud83d\\ud83d", "\\", "\x01", "\x1f", "\x7f"}}; + const std::array utf8 = {{"\xc3\xa4", "\xe2\x82\xac", "\xf0\x9f\x98\x80", "\xc3", "\xff", "\x80"}}; + std::mt19937 rng(2029); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed): reproducible + for (std::size_t run = 0; run <= 40; ++run) + { + for (int i = 0; i < 30; ++i) + { + std::string content = std::string(rng() % 20, 'a'); + const auto pieces = 1 + (rng() % 6); + for (unsigned k = 0; k < pieces; ++k) + { + content += valid[rng() % valid.size()]; + content += std::string(k == 0 ? run : rng() % 41, static_cast('b' + (rng() % 20))); + switch (rng() % 8) + { + case 0: + content += utf8[rng() % 3]; + break; + case 1: + content += utf8[rng() % utf8.size()]; + break; + case 2: + content += invalid[rng() % invalid.size()]; + break; + default: + break; + } + } + for (const std::string& text : + { + "[\"" + content + "\"]", "{\"" + content + "\":1}", "[\"" + content, "[\"" + content.substr(0, content.size() - (rng() % 4)) + "\"]" + }) + { + check_accept(text); + } + } + } + } + + SECTION("escapes in a row") + { + for (const char* content : + { + "\\n", R"(\n\t\\)", R"(\u00e9\u00e9\u4e2d)", "\\u00e9\\u00e", "\\u00e9\\", "\\n\\q", R"(\n\ud83d\ude00\n)", + R"(\ud83d\ude00\ud83d)", R"(\n\n\n abc \n\n)", "\\n\x01", "\\n\xC3\x28", "\\n\\" + }) + { + check_accept(std::string("[\"") + content + "\"]"); + check_accept(std::string("[\"") + content); + check_accept(std::string("{\"") + content + "\":1}"); + } + } + SECTION("test files") { for (const char* name : diff --git a/tests/src/unit-json_view_macros.cpp b/tests/src/unit-json_view_macros.cpp index 2cc8b7744..d2c9e197f 100644 --- a/tests/src/unit-json_view_macros.cpp +++ b/tests/src/unit-json_view_macros.cpp @@ -25,7 +25,7 @@ #endif // the view's own macros do not leak -#if defined(NLOHMANN_VIEW_LIKELY) || defined(NLOHMANN_VIEW_UNLIKELY) || defined(NLOHMANN_VIEW_ALWAYS_INLINE) || defined(NLOHMANN_VIEW_NOINLINE) \ +#if defined(NLOHMANN_VIEW_LIKELY) || defined(NLOHMANN_VIEW_UNLIKELY) || defined(NLOHMANN_VIEW_ALWAYS_INLINE) || defined(NLOHMANN_VIEW_NOINLINE) || defined(NLOHMANN_VIEW_NOCLONE) \ || defined(NLOHMANN_VIEW_NODISCARD) || defined(NLOHMANN_VIEW_THROW) || defined(NLOHMANN_VIEW_HAS_CPP_17) || defined(NLOHMANN_VIEW_LITTLE_ENDIAN) \ || defined(NLOHMANN_VIEW_REPEAT16) #error "json_view.hpp leaks a macro" From 162eda2574bb46a6d31c92a0037d9939f1009f84 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 1 Nov 2026 17:21:19 +0100 Subject: [PATCH 09/54] json_view: skip the digits of a fraction 16 bytes at a time (SSE2) (#5971) Signed-off-by: Niels Lohmann --- include/nlohmann/detail/view/builder.hpp | 2 +- include/nlohmann/detail/view/scan.hpp | 51 +++++++++++++++++++++++ single_include/nlohmann/json_view.hpp | 53 +++++++++++++++++++++++- tests/src/unit-json_view_builder.cpp | 48 +++++++++++++++++++++ 4 files changed, 152 insertions(+), 2 deletions(-) diff --git a/include/nlohmann/detail/view/builder.hpp b/include/nlohmann/detail/view/builder.hpp index 4b729ce39..938d6d73b 100644 --- a/include/nlohmann/detail/view/builder.hpp +++ b/include/nlohmann/detail/view/builder.hpp @@ -1189,7 +1189,7 @@ indent_done: { ++p; const unsigned char* const f0 = p; - p = skip_digits(p, e); + p = skip_fraction_digits(p, e); if (NLOHMANN_VIEW_UNLIKELY(p == f0)) { return fail(error_code::number_after_dot); diff --git a/include/nlohmann/detail/view/scan.hpp b/include/nlohmann/detail/view/scan.hpp index b8d862203..9cea6b05c 100644 --- a/include/nlohmann/detail/view/scan.hpp +++ b/include/nlohmann/detail/view/scan.hpp @@ -198,6 +198,57 @@ inline const unsigned char* find_block_comment_end(const unsigned char* p, const return p; } +/// The digits of a fraction, whose number varies from one number to the next: skip_digits() with 16 bytes compared at +/// once and the end of the run taken from a bit mask (SSE2), instead of a compare and a branch for each digit. That +/// is about a third of the instructions for a fraction of 12 digits, and no branch that is mispredicted when the +/// lengths vary. Where there is no SSE2, skip_digits() is used. +#if NLOHMANN_VIEW_SSE2 +// The last bytes of the input, one at a time. GCC unrolls this loop in every inlined copy of the function below, which +// makes the parse loop 4 KiB larger and slows down documents that have hardly any fractions; out of line, it does +// not. Clang does better with the loop inline (a call in the parse loop costs it more than the unrolling). +#if defined(__GNUC__) && !defined(__clang__) +inline NLOHMANN_VIEW_NOINLINE const unsigned char* skip_digits_tail(const unsigned char* p, const unsigned char* e) noexcept +{ + while (p != e && is_digit(*p)) + { + ++p; + } + return p; +} +#else +NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* skip_digits_tail(const unsigned char* p, const unsigned char* e) noexcept +{ + while (p != e && is_digit(*p)) + { + ++p; + } + return p; +} +#endif + +NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* skip_fraction_digits(const unsigned char* p, const unsigned char* e) noexcept +{ + while (e - p >= 16) + { + const __m128i in = _mm_loadu_si128(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) + // a digit is above '/' and below ':' (signed compares: a byte with the top bit set is negative, and neither) + const __m128i digits = _mm_and_si128(_mm_cmpgt_epi8(in, _mm_set1_epi8('/')), _mm_cmplt_epi8(in, _mm_set1_epi8(':'))); + const auto not_digit = static_cast(~static_cast(_mm_movemask_epi8(digits)) & 0xFFFFu); + if (not_digit != 0) + { + return p + count_trailing_zeros(not_digit); + } + p += 16; + } + return skip_digits_tail(p, e); +} +#else +NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* skip_fraction_digits(const unsigned char* p, const unsigned char* e) noexcept +{ + return skip_digits(p, e); +} +#endif + /// powers of ten up to 10^19 as integers inline std::uint64_t int_pow10(unsigned k) noexcept { diff --git a/single_include/nlohmann/json_view.hpp b/single_include/nlohmann/json_view.hpp index f446b273d..408d845d4 100644 --- a/single_include/nlohmann/json_view.hpp +++ b/single_include/nlohmann/json_view.hpp @@ -1230,6 +1230,57 @@ inline const unsigned char* find_block_comment_end(const unsigned char* p, const return p; } +/// The digits of a fraction, whose number varies from one number to the next: skip_digits() with 16 bytes compared at +/// once and the end of the run taken from a bit mask (SSE2), instead of a compare and a branch for each digit. That +/// is about a third of the instructions for a fraction of 12 digits, and no branch that is mispredicted when the +/// lengths vary. Where there is no SSE2, skip_digits() is used. +#if NLOHMANN_VIEW_SSE2 +// The last bytes of the input, one at a time. GCC unrolls this loop in every inlined copy of the function below, which +// makes the parse loop 4 KiB larger and slows down documents that have hardly any fractions; out of line, it does +// not. Clang does better with the loop inline (a call in the parse loop costs it more than the unrolling). +#if defined(__GNUC__) && !defined(__clang__) +inline NLOHMANN_VIEW_NOINLINE const unsigned char* skip_digits_tail(const unsigned char* p, const unsigned char* e) noexcept +{ + while (p != e && is_digit(*p)) + { + ++p; + } + return p; +} +#else +NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* skip_digits_tail(const unsigned char* p, const unsigned char* e) noexcept +{ + while (p != e && is_digit(*p)) + { + ++p; + } + return p; +} +#endif + +NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* skip_fraction_digits(const unsigned char* p, const unsigned char* e) noexcept +{ + while (e - p >= 16) + { + const __m128i in = _mm_loadu_si128(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) + // a digit is above '/' and below ':' (signed compares: a byte with the top bit set is negative, and neither) + const __m128i digits = _mm_and_si128(_mm_cmpgt_epi8(in, _mm_set1_epi8('/')), _mm_cmplt_epi8(in, _mm_set1_epi8(':'))); + const auto not_digit = static_cast(~static_cast(_mm_movemask_epi8(digits)) & 0xFFFFu); + if (not_digit != 0) + { + return p + count_trailing_zeros(not_digit); + } + p += 16; + } + return skip_digits_tail(p, e); +} +#else +NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* skip_fraction_digits(const unsigned char* p, const unsigned char* e) noexcept +{ + return skip_digits(p, e); +} +#endif + /// powers of ten up to 10^19 as integers inline std::uint64_t int_pow10(unsigned k) noexcept { @@ -2443,7 +2494,7 @@ indent_done: { ++p; const unsigned char* const f0 = p; - p = skip_digits(p, e); + p = skip_fraction_digits(p, e); if (NLOHMANN_VIEW_UNLIKELY(p == f0)) { return fail(error_code::number_after_dot); diff --git a/tests/src/unit-json_view_builder.cpp b/tests/src/unit-json_view_builder.cpp index 418f219da..06059aba8 100644 --- a/tests/src/unit-json_view_builder.cpp +++ b/tests/src/unit-json_view_builder.cpp @@ -668,6 +668,54 @@ TEST_CASE("json_view builder: strings across vector blocks") } } +TEST_CASE("json_view builder: fraction digits") +{ + // The digits of a fraction are skipped 16 bytes at a time where there are + // that many left (SSE2), and one at a time after that. Fractions of every + // length from 0 to 40 digits, followed by every kind of byte that ends a + // number or lies next to the digits in ASCII ('/' and ':'), as values, as + // keys' neighbours, and at the very end of the text (a vector without a + // terminating NUL). The length of the number is part of the value that + // the test compares with json::parse(). + const std::string followers = ",]}/:.eE+-x \n\t\"09"; + std::mt19937 rng(2028); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed): reproducible + for (std::size_t digits = 0; digits <= 40; ++digits) + { + for (int i = 0; i < 3; ++i) + { + std::string fraction; + for (std::size_t k = 0; k < digits; ++k) + { + fraction += static_cast('0' + (rng() % 10)); + } + const std::string number = std::string(rng() % 2 == 0 ? "-" : "") + std::to_string(rng() % 100000) + "." + fraction; + check_same(number); + check_same("[" + number + "]"); + std::string pair; + pair.reserve(number.size() * 2 + 3); + pair += '['; + pair += number; + pair += ','; + pair += number; + pair += ']'; + check_same(pair); + check_same("{\"a\":" + number + "}"); + for (const char follower : followers) + { + check_same(number + follower); + check_same("[" + number + follower + "]"); + check_same("[" + number + follower + std::string(rng() % 20, '1')); + } + // everything near the end of the text: the number cut in every place + for (std::size_t cut = 1; cut <= 18 && cut <= number.size(); ++cut) + { + check_same(number.substr(0, number.size() - cut)); + check_same("[" + number.substr(0, number.size() - cut)); + } + } + } +} + TEST_CASE("json_view builder: escapes between runs of text") { // The text between escapes is copied 16 bytes at a time where there are From 5337d841d43bbc2fd9e54ea3367610ca49ad22e8 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 1 Nov 2026 17:21:31 +0100 Subject: [PATCH 10/54] Document and test the lifetime of string_view members decoded from a json_view (#5973) Signed-off-by: Niels Lohmann --- docs/mkdocs/docs/features/json_view.md | 28 +++++++++++ tests/src/unit-json_view_from_json.cpp | 66 ++++++++++++++++++++++++++ 2 files changed, 94 insertions(+) diff --git a/docs/mkdocs/docs/features/json_view.md b/docs/mkdocs/docs/features/json_view.md index 09779dba2..08529fd7d 100644 --- a/docs/mkdocs/docs/features/json_view.md +++ b/docs/mkdocs/docs/features/json_view.md @@ -253,6 +253,34 @@ Two conversions never copy at all: Both results are only valid as long as the view — and, for a string with no escapes, the borrowed source text — is. +### `string_view` members + +A struct whose member is a `#!cpp std::string_view` (C++17) can be decoded from a view with the +`NLOHMANN_DEFINE_TYPE_*` macros, and the member is not copied either: for a string without escape sequences it points +into the document's source text, and for a string with escape sequences into the document's buffer of decoded strings. +The member is valid as long as the document is alive and, for a document that +[borrows its source text](#ownership-and-lifetime) (as [`parse()`](../api/basic_json_document/parse.md) does for +an lvalue `#!cpp std::string`, a `#!cpp std::string_view`, or a C string), as long as that text is. + +```cpp +struct record +{ + std::string_view name; + int count; + NLOHMANN_DEFINE_TYPE_INTRUSIVE(record, name, count) +}; + +const std::string text = R"({"name": "widget", "count": 3})"; +const json_document doc = json_document::parse(text); // borrows text +const record r = doc.root().get(); // r.name points into text, valid while text and doc are +``` + +!!! warning "Decoding into a `string_view` from a `json` value dangles at once" + + The same struct also compiles with `#!cpp json::parse(text).get()`, but there `name` points into the + temporary `json` value, which is destroyed at the end of the statement. Use a named + [`json_document`](../api/json_document.md), as above, to decode such a struct. + ## Writing generic code A view's type names its `BasicJsonType` and its [`view_mode`](../api/view_mode.md): diff --git a/tests/src/unit-json_view_from_json.cpp b/tests/src/unit-json_view_from_json.cpp index a1d94d586..fa01b639d 100644 --- a/tests/src/unit-json_view_from_json.cpp +++ b/tests/src/unit-json_view_from_json.cpp @@ -39,6 +39,7 @@ using nlohmann::ordered_json_document; #ifdef JSON_HAS_CPP_17 #include + #include #endif // NLOHMANN_JSON_SERIALIZE_ENUM uses a static std::pair; same() compares floating-point values on purpose @@ -493,5 +494,70 @@ TEST_CASE("json_view from_json") #endif } +#ifdef JSON_HAS_CPP_17 +namespace view_from_json +{ +// a string_view member is not copied: it points into the source text or, for a +// string with escape sequences, into the buffer of decoded strings of the document +struct named_view +{ + // initialized for GCC -Weffc++ + // NOLINTNEXTLINE(readability-redundant-member-init) + std::string_view name{}; + int n = 0; + NLOHMANN_DEFINE_TYPE_INTRUSIVE(named_view, name, n) +}; +} // namespace view_from_json + +TEST_CASE("json_view from_json: string_view members borrow from the document") +{ + using view_from_json::named_view; + + // whether all of s is inside the bytes of text + const auto within = [](std::string_view s, std::string_view text) + { + const auto first = reinterpret_cast(text.data()); + const auto begin = reinterpret_cast(s.data()); + return begin >= first && begin + s.size() <= first + text.size(); + }; + + SECTION("a string without escapes points into the source text") + { + const std::string text = R"({"name": "hello", "n": 1})"; + const json_document d = json_document::parse(text); + REQUIRE_FALSE(d.owns_source()); + const auto r = d.root().get(); + CHECK(r.name == "hello"); + CHECK(r.n == 1); + CHECK(within(r.name, text)); + CHECK(within(r.name, d.source())); + } + + SECTION("a string with escapes points into the document, not into the source text") + { + const std::string text = R"({"name": "a\nb", "n": 2})"; + json_document d = json_document::parse(text); + REQUIRE_FALSE(d.owns_source()); + const auto r = d.root().get(); + CHECK(r.name == "a\nb"); + CHECK(r.n == 2); + CHECK_FALSE(within(r.name, text)); + // valid as long as the document is alive, even if the document is moved + const json_document moved = std::move(d); + CHECK(r.name == "a\nb"); + CHECK(moved.root().get().name == r.name); + } + + SECTION("a document that owns its text keeps the members readable while it is alive") + { + const json_document d = json_document::parse(std::string(R"({"name": "owned", "n": 3})")); + REQUIRE(d.owns_source()); + const auto r = d.root().get(); + CHECK(r.name == "owned"); + CHECK(within(r.name, d.source())); + } +} +#endif + DOCTEST_CLANG_SUPPRESS_WARNING_POP DOCTEST_GCC_SUPPRESS_WARNING_POP From 6c3ab966d3b086657301ca14c54d5839b38c3e8d Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 1 Nov 2026 17:22:31 +0100 Subject: [PATCH 11/54] CBOR reader: read runs of scalar array elements in one loop (#5975) * CBOR reader: read runs of scalar array elements in one loop When a definite-length CBOR array opens and the input is contiguous, read the integers, floats, booleans, and nulls that follow in a tight loop instead of going through parse_cbor_internal's container logic and parse_cbor_value's dispatch for every element. The loop stops at any other kind of value and when fewer than 9 bytes are left; those values are read by the general path exactly as before. Non-contiguous inputs are not affected (tag dispatch on the bulk-scan capability). Also pass a member empty string as the number source to number_float instead of constructing a std::string for every float. Signed-off-by: Niels Lohmann * CBOR reader: address review of the scalar-run reader - parse_cbor_scalar_run: set the size explicitly in every case that reads a value, instead of relying on a default that every other case overwrites. - Remove cbor_scalar_follows(): parse_cbor_scalar_run already returns on the same conditions, and without the pre-check SAX canada needs another 8.5 % fewer instructions with GCC 13 (0.915 against the previous version), with no row more than 0.5 % worse (Mac and GCC). - Restore the comment on why parse_cbor_internal copies the container frame. Signed-off-by: Niels Lohmann * CBOR reader: enter the scalar run only when a scalar follows An array of containers or strings called the out-of-line run start once per array only to leave it at once; on nested data (arrays of objects) that call made reading slower than before the run reader. enter_cbor_array now tests the first byte and the remaining length inline and calls the run only for a scalar, so the run's loop condition moves to the end (do-while). Signed-off-by: Niels Lohmann * CBOR reader: mark the initial accepted in the scalar run as a false-positive dead store ci_infer reports DEAD_STORE for 'bool accepted = true;' in parse_cbor_scalar_run because every case assigns accepted before it is read. Signed-off-by: Niels Lohmann * CBOR reader: drop the scalar pre-check before the run enter_cbor_array calls the run directly again, and parse_cbor_scalar_run tests the remaining count and length at the top of a while loop; its own switch already leaves on any value that is not a scalar. The MessagePack reader is built the same way. Instructions per repetition, new/old: clang SAX canada 0.983, read canada 0.993; GCC 13 read canada 0.997, the other rows at most 1.004. Signed-off-by: Niels Lohmann * CBOR reader: say why the initial accepted in the scalar run stays Infer's DEAD_STORE report is right, not a false positive: every case that reaches the read assigns accepted. The initializer is kept for clang-tidy's cppcoreguidelines-init-variables. Signed-off-by: Niels Lohmann * CBOR reader: give the scalar run the bulk tag instead of a wrapper parse_cbor_scalar_run takes the bulk-scan tag itself and is the function that is kept out of line, so the one-line parse_cbor_array_start wrapper goes, as the UBJSON typed run already does. Signed-off-by: Niels Lohmann * CBOR reader: start the scalar run's accepted at false Infer ignores dead stores of 0 (--liveness-ignored-constant defaults to "0"), and false is 0 to it, so the suppression and its comment go. Signed-off-by: Niels Lohmann * CBOR reader: let the scalar run find its array itself parse_cbor_scalar_run reads container_stack.back() in its bulk version, so the caller no longer evaluates it for input that is not contiguous. Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann --- .../nlohmann/detail/input/binary_reader.hpp | 184 ++++++++++++++++-- single_include/nlohmann/json.hpp | 184 ++++++++++++++++-- tests/src/unit-cbor.cpp | 74 +++++++ 3 files changed, 414 insertions(+), 28 deletions(-) diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index b671fd84e..40df891bc 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -745,11 +745,13 @@ class binary_reader bool get_cbor_negative_integer() { NumberType number{}; - if (JSON_HEDLEY_UNLIKELY(!get_number(number))) - { - return false; - } + return get_number(number) && emit_cbor_negative_integer(number); + } + /// pass the CBOR negative integer -1 - @a number to the SAX parser + template + bool emit_cbor_negative_integer(const NumberType number) + { // the value is -1 - number, which fits into number_integer_t // whenever number does; the outer cast undoes the integral promotion // for number_integer_t types narrower than int @@ -974,7 +976,7 @@ class binary_reader case 0x95: case 0x96: case 0x97: - return enter_array(conditional_static_cast(static_cast(current) & 0x1Fu)); + return enter_cbor_array(conditional_static_cast(static_cast(current) & 0x1Fu)); case 0x98: // array (one-byte uint8_t for n follows) case 0x99: // array (two-byte uint16_t for n follow) @@ -983,7 +985,7 @@ class binary_reader { std::uint64_t len{}; std::size_t size{}; - return get_cbor_argument(len) && get_cbor_container_size(len, size, "array") && enter_array(size); + return get_cbor_argument(len) && get_cbor_container_size(len, size, "array") && enter_cbor_array(size); } case 0x9F: // array (indefinite length) @@ -1680,6 +1682,155 @@ class binary_reader return true; } + /*! + @brief read a big-endian number of type @a NumberType at @a data + */ + template + NumberType load_big_endian(const unsigned char* data) const noexcept + { + NumberType result{}; + std::memcpy(&result, data, sizeof(NumberType)); + if (is_little_endian) + { + byte_swap(result); + } + return result; + } + + /*! + @brief read the integers, floats, booleans, and nulls that follow in a + definite-length array, without the per-element bookkeeping of + @ref parse_cbor_internal + + Stops at the end of the array, at any other kind of value, and when fewer + than 9 bytes (the longest of these values) are left in the input, leaving + those to @ref parse_cbor_value, which reads them the same way. + + Out of line, so that @ref parse_cbor_value stays small enough to be inlined. + + The array is the one on top of @ref container_stack; its count of + elements not yet read is updated. + + @return whether the SAX parser accepted the values + */ + JSON_HEDLEY_NEVER_INLINE + bool parse_cbor_scalar_run(std::true_type /*bulk*/) + { + // nothing pushes onto the stack while the run reads scalars + std::size_t& remaining = container_stack.back().remaining; + while (remaining != 0 && ia.bulk_remaining() >= 9) + { + const auto* const data = reinterpret_cast(ia.bulk_data()); + const unsigned char head = data[0]; + std::size_t size = 0; + bool accepted = false; + switch (head) + { + case 0x18: + size = 2; + accepted = emit_unsigned(data[1]); + break; + case 0x19: + size = 3; + accepted = emit_unsigned(load_big_endian(data + 1)); + break; + case 0x1A: + size = 5; + accepted = emit_unsigned(load_big_endian(data + 1)); + break; + case 0x1B: + size = 9; + accepted = emit_unsigned(load_big_endian(data + 1)); + break; + case 0x38: + size = 2; + accepted = emit_cbor_negative_integer(data[1]); + break; + case 0x39: + size = 3; + accepted = emit_cbor_negative_integer(load_big_endian(data + 1)); + break; + case 0x3A: + size = 5; + accepted = emit_cbor_negative_integer(load_big_endian(data + 1)); + break; + case 0x3B: + size = 9; + accepted = emit_cbor_negative_integer(load_big_endian(data + 1)); + break; + case 0xF4: + size = 1; + accepted = sax->boolean(false); + break; + case 0xF5: + size = 1; + accepted = sax->boolean(true); + break; + case 0xF6: + size = 1; + accepted = sax->null(); + break; + case 0xFA: + size = 5; + accepted = emit_float(load_big_endian(data + 1)); + break; + case 0xFB: + size = 9; + accepted = emit_float(load_big_endian(data + 1)); + break; + default: + if (head <= 0x17) + { + size = 1; + accepted = sax->number_unsigned(static_cast(head)); + } + else if (head >= 0x20 && head <= 0x37) + { + size = 1; + accepted = sax->number_integer(static_cast(0x20 - 1 - head)); + } + else + { + // any other value is read by parse_cbor_value + return true; + } + break; + } + // as if read by get() and get_number(): current is the head byte + ia.bulk_skip(size); + chars_read += size; + current = head; + --remaining; + if (JSON_HEDLEY_UNLIKELY(!accepted)) + { + return false; + } + } + return true; + } + + /*! + @brief open a definite-length array and read the scalars it begins with + (see @ref parse_cbor_scalar_run) + + Only the start of an array is tried, which covers arrays of numbers + (e.g. coordinates) but costs arrays of other values only one call, not one + per element. + + @param[in] len the number of elements + @return whether the SAX parser accepted the array and the values read + */ + bool enter_cbor_array(const std::size_t len) + { + return enter_array(len) && (len == 0 || parse_cbor_scalar_run(std::integral_constant {})); + } + + /// input that is not contiguous: every value is read by parse_cbor_value + bool parse_cbor_scalar_run(std::false_type /*bulk*/) const noexcept + { + return true; + } + /*! @brief read a CBOR value and everything nested inside it @@ -1709,6 +1860,7 @@ class binary_reader // pop_back() below, which destroys the container_stack element // it would otherwise alias const container_frame top = container_stack.back(); + const bool is_object = top.is_object; bool at_end = false; if (top.remaining != npos) @@ -1720,7 +1872,7 @@ class binary_reader { // claim the element about to be read --container_stack.back().remaining; - if (top.is_object) + if (is_object) { get(); } @@ -1733,7 +1885,7 @@ class binary_reader // Testing for it consumes a byte, which is the first byte // of the next element when it is not one. at_end = (get() == 0xFF); - fetch = top.is_object; + fetch = is_object; } if (at_end) @@ -1750,7 +1902,7 @@ class binary_reader continue; } - if (top.is_object) + if (is_object) { if (JSON_HEDLEY_UNLIKELY(!skip_cbor_key_tags(tag_handler))) { @@ -3695,13 +3847,13 @@ class binary_reader return sax->null(); case 0xFB: - return sax->number_float(static_cast(-1.0), ""); + return sax->number_float(static_cast(-1.0), no_source); case 0xFC: - return sax->number_float(static_cast(0.0), ""); + return sax->number_float(static_cast(0.0), no_source); case 0xFD: - return sax->number_float(static_cast(1.0), ""); + return sax->number_float(static_cast(1.0), no_source); case 0xFF: // empty string { @@ -4205,7 +4357,7 @@ class binary_reader { return out_of_range_error(exception_id::number_overflow, "number overflow", "value"); } - return sax->number_float(result, ""); + return sax->number_float(result, no_source); } /*! @@ -4265,7 +4417,7 @@ class binary_reader }(); return sax->number_float((half & 0x8000u) != 0 ? static_cast(-val) - : static_cast(val), ""); + : static_cast(val), no_source); } /*! @@ -4590,6 +4742,10 @@ class binary_reader /// the SAX parser json_sax_t* sax = nullptr; + /// the (empty) source text of every number_float event: binary formats + /// have none, and a string_t built for each float would cost more + const string_t no_source{}; + /// the containers that have been opened and not closed yet; see @ref container_frame std::vector container_stack{}; diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index e2ead9c41..d16cbb3d4 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -18549,11 +18549,13 @@ class binary_reader bool get_cbor_negative_integer() { NumberType number{}; - if (JSON_HEDLEY_UNLIKELY(!get_number(number))) - { - return false; - } + return get_number(number) && emit_cbor_negative_integer(number); + } + /// pass the CBOR negative integer -1 - @a number to the SAX parser + template + bool emit_cbor_negative_integer(const NumberType number) + { // the value is -1 - number, which fits into number_integer_t // whenever number does; the outer cast undoes the integral promotion // for number_integer_t types narrower than int @@ -18778,7 +18780,7 @@ class binary_reader case 0x95: case 0x96: case 0x97: - return enter_array(conditional_static_cast(static_cast(current) & 0x1Fu)); + return enter_cbor_array(conditional_static_cast(static_cast(current) & 0x1Fu)); case 0x98: // array (one-byte uint8_t for n follows) case 0x99: // array (two-byte uint16_t for n follow) @@ -18787,7 +18789,7 @@ class binary_reader { std::uint64_t len{}; std::size_t size{}; - return get_cbor_argument(len) && get_cbor_container_size(len, size, "array") && enter_array(size); + return get_cbor_argument(len) && get_cbor_container_size(len, size, "array") && enter_cbor_array(size); } case 0x9F: // array (indefinite length) @@ -19484,6 +19486,155 @@ class binary_reader return true; } + /*! + @brief read a big-endian number of type @a NumberType at @a data + */ + template + NumberType load_big_endian(const unsigned char* data) const noexcept + { + NumberType result{}; + std::memcpy(&result, data, sizeof(NumberType)); + if (is_little_endian) + { + byte_swap(result); + } + return result; + } + + /*! + @brief read the integers, floats, booleans, and nulls that follow in a + definite-length array, without the per-element bookkeeping of + @ref parse_cbor_internal + + Stops at the end of the array, at any other kind of value, and when fewer + than 9 bytes (the longest of these values) are left in the input, leaving + those to @ref parse_cbor_value, which reads them the same way. + + Out of line, so that @ref parse_cbor_value stays small enough to be inlined. + + The array is the one on top of @ref container_stack; its count of + elements not yet read is updated. + + @return whether the SAX parser accepted the values + */ + JSON_HEDLEY_NEVER_INLINE + bool parse_cbor_scalar_run(std::true_type /*bulk*/) + { + // nothing pushes onto the stack while the run reads scalars + std::size_t& remaining = container_stack.back().remaining; + while (remaining != 0 && ia.bulk_remaining() >= 9) + { + const auto* const data = reinterpret_cast(ia.bulk_data()); + const unsigned char head = data[0]; + std::size_t size = 0; + bool accepted = false; + switch (head) + { + case 0x18: + size = 2; + accepted = emit_unsigned(data[1]); + break; + case 0x19: + size = 3; + accepted = emit_unsigned(load_big_endian(data + 1)); + break; + case 0x1A: + size = 5; + accepted = emit_unsigned(load_big_endian(data + 1)); + break; + case 0x1B: + size = 9; + accepted = emit_unsigned(load_big_endian(data + 1)); + break; + case 0x38: + size = 2; + accepted = emit_cbor_negative_integer(data[1]); + break; + case 0x39: + size = 3; + accepted = emit_cbor_negative_integer(load_big_endian(data + 1)); + break; + case 0x3A: + size = 5; + accepted = emit_cbor_negative_integer(load_big_endian(data + 1)); + break; + case 0x3B: + size = 9; + accepted = emit_cbor_negative_integer(load_big_endian(data + 1)); + break; + case 0xF4: + size = 1; + accepted = sax->boolean(false); + break; + case 0xF5: + size = 1; + accepted = sax->boolean(true); + break; + case 0xF6: + size = 1; + accepted = sax->null(); + break; + case 0xFA: + size = 5; + accepted = emit_float(load_big_endian(data + 1)); + break; + case 0xFB: + size = 9; + accepted = emit_float(load_big_endian(data + 1)); + break; + default: + if (head <= 0x17) + { + size = 1; + accepted = sax->number_unsigned(static_cast(head)); + } + else if (head >= 0x20 && head <= 0x37) + { + size = 1; + accepted = sax->number_integer(static_cast(0x20 - 1 - head)); + } + else + { + // any other value is read by parse_cbor_value + return true; + } + break; + } + // as if read by get() and get_number(): current is the head byte + ia.bulk_skip(size); + chars_read += size; + current = head; + --remaining; + if (JSON_HEDLEY_UNLIKELY(!accepted)) + { + return false; + } + } + return true; + } + + /*! + @brief open a definite-length array and read the scalars it begins with + (see @ref parse_cbor_scalar_run) + + Only the start of an array is tried, which covers arrays of numbers + (e.g. coordinates) but costs arrays of other values only one call, not one + per element. + + @param[in] len the number of elements + @return whether the SAX parser accepted the array and the values read + */ + bool enter_cbor_array(const std::size_t len) + { + return enter_array(len) && (len == 0 || parse_cbor_scalar_run(std::integral_constant {})); + } + + /// input that is not contiguous: every value is read by parse_cbor_value + bool parse_cbor_scalar_run(std::false_type /*bulk*/) const noexcept + { + return true; + } + /*! @brief read a CBOR value and everything nested inside it @@ -19513,6 +19664,7 @@ class binary_reader // pop_back() below, which destroys the container_stack element // it would otherwise alias const container_frame top = container_stack.back(); + const bool is_object = top.is_object; bool at_end = false; if (top.remaining != npos) @@ -19524,7 +19676,7 @@ class binary_reader { // claim the element about to be read --container_stack.back().remaining; - if (top.is_object) + if (is_object) { get(); } @@ -19537,7 +19689,7 @@ class binary_reader // Testing for it consumes a byte, which is the first byte // of the next element when it is not one. at_end = (get() == 0xFF); - fetch = top.is_object; + fetch = is_object; } if (at_end) @@ -19554,7 +19706,7 @@ class binary_reader continue; } - if (top.is_object) + if (is_object) { if (JSON_HEDLEY_UNLIKELY(!skip_cbor_key_tags(tag_handler))) { @@ -21499,13 +21651,13 @@ class binary_reader return sax->null(); case 0xFB: - return sax->number_float(static_cast(-1.0), ""); + return sax->number_float(static_cast(-1.0), no_source); case 0xFC: - return sax->number_float(static_cast(0.0), ""); + return sax->number_float(static_cast(0.0), no_source); case 0xFD: - return sax->number_float(static_cast(1.0), ""); + return sax->number_float(static_cast(1.0), no_source); case 0xFF: // empty string { @@ -22009,7 +22161,7 @@ class binary_reader { return out_of_range_error(exception_id::number_overflow, "number overflow", "value"); } - return sax->number_float(result, ""); + return sax->number_float(result, no_source); } /*! @@ -22069,7 +22221,7 @@ class binary_reader }(); return sax->number_float((half & 0x8000u) != 0 ? static_cast(-val) - : static_cast(val), ""); + : static_cast(val), no_source); } /*! @@ -22394,6 +22546,10 @@ class binary_reader /// the SAX parser json_sax_t* sax = nullptr; + /// the (empty) source text of every number_float event: binary formats + /// have none, and a string_t built for each float would cost more + const string_t no_source{}; + /// the containers that have been opened and not closed yet; see @ref container_frame std::vector container_stack{}; diff --git a/tests/src/unit-cbor.cpp b/tests/src/unit-cbor.cpp index 44cfa7723..930cbe7f5 100644 --- a/tests/src/unit-cbor.cpp +++ b/tests/src/unit-cbor.cpp @@ -29,7 +29,9 @@ using nlohmann::json; #include "round_trip_corpus.hpp" #include "test_utils.hpp" #include "sax_countdown.hpp" +#include "sax_event_loggers.hpp" using utils::SaxCountdown; +using utils::SaxEventLogger; TEST_CASE("CBOR") @@ -2446,6 +2448,78 @@ TEST_CASE("CBOR SAX parsing stops at every event") CHECK(count_events(std::vector({0xBF, 0x61, 'a', 0x9F, 0x01, 0xFF, 0xFF})) == 6); } +TEST_CASE("CBOR arrays of scalars from contiguous and non-contiguous input") +{ + // contiguous input reads the integers, floats, booleans, and nulls an + // array begins with in one loop, other input value by value; both must + // give the same events, value, or error for the complete document and for + // every truncation of it, and stop at every event a SAX handler rejects + const std::vector cbor = + { + 0xA3, // map(3) + 0x61, 'a', // "a" + 0x98, 0x16, // array(22) + 0x00, 0x17, 0x18, 0xFF, 0x19, 0x01, 0x00, // 0, 23, 255, 256 + 0x1A, 0x00, 0x01, 0x00, 0x00, // 65536 + 0x1B, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 18446744073709551615 + 0x20, 0x37, 0x38, 0xFF, 0x39, 0x01, 0x00, // -1, -24, -256, -257 + 0x3A, 0x00, 0x01, 0x00, 0x00, // -65537 + 0x3B, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // -18446744073709551616 (a float) + 0x3B, 0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // -9223372036854775808 + 0xF4, 0xF5, 0xF6, // false, true, null + 0xFA, 0x3F, 0xC0, 0x00, 0x00, // 1.5 (float32) + 0xFB, 0xBF, 0xB9, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9A, // -0.1 + 0xF9, 0x3C, 0x00, // 1.0 (float16, ends the loop) + 0x01, 0x61, 'a', // 1, "a" + 0xFB, 0x40, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 2.5 + 0x61, 'b', // "b" + 0x83, // array(3) + 0x82, 0xFB, 0x3F, 0xF4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // [1.25, + 0xFB, 0xC0, 0x0E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // -3.75] + 0x82, 0xFB, 0x3F, 0xF4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // [1.25, + 0xFB, 0xC0, 0x0E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // -3.75] + 0x80, // [] + 0x61, 'c', // "c" + 0x84, 0xA0, 0x01, 0xC1, 0x02, 0x81, 0x03 // [{}, 1, tag(1) 2, [3]] + }; + const auto tags = json::cbor_tag_handler_t::ignore; + + const json j = json::from_cbor(cbor, true, true, tags); + CHECK(j["a"].size() == 22); + CHECK(j["a"][5] == 18446744073709551615u); + CHECK(j["a"][11].is_number_float()); + CHECK(j["a"][11] == -18446744073709551616.0); + CHECK(j["a"][12] == (std::numeric_limits::min)()); + CHECK(j["a"][17] == -0.1); + CHECK(j["b"] == json::parse("[[1.25, -3.75], [1.25, -3.75], []]")); + CHECK(j["c"] == json::parse("[{}, 1, 2, [3]]")); + + for (std::size_t length = 0; length <= cbor.size(); ++length) + { + CAPTURE(length) + const std::vector contiguous(cbor.begin(), cbor.begin() + static_cast(length)); + const std::list bytewise(contiguous.begin(), contiguous.end()); + + CHECK(json::from_cbor(contiguous, true, false, tags).dump() == json::from_cbor(bytewise.begin(), bytewise.end(), true, false, tags).dump()); + + SaxEventLogger from_contiguous; + SaxEventLogger from_bytewise; + CHECK(json::sax_parse(contiguous, &from_contiguous, json::input_format_t::cbor, true, false, false, tags) + == json::sax_parse(bytewise.begin(), bytewise.end(), &from_bytewise, json::input_format_t::cbor, true, false, false, tags)); + CHECK(from_contiguous.events == from_bytewise.events); + } + + for (int events = 0; events <= 60; ++events) + { + CAPTURE(events) + const std::list bytewise(cbor.begin(), cbor.end()); + SaxCountdown from_contiguous(events); + SaxCountdown from_bytewise(events); + CHECK(json::sax_parse(cbor, &from_contiguous, json::input_format_t::cbor, true, false, false, tags) + == json::sax_parse(bytewise.begin(), bytewise.end(), &from_bytewise, json::input_format_t::cbor, true, false, false, tags)); + } +} + TEST_CASE("CBOR indefinite-length strings do not recurse per chunk") { // Reading an indefinite-length string or byte array used to call itself From aecbef39490cfea51203f9663ce06e00a3b3796f Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 1 Nov 2026 17:22:32 +0100 Subject: [PATCH 12/54] Binary readers: swap the bytes of floats with a single byteswap (#5976) * Binary readers: swap the bytes of floats with a single byteswap byte_swap reversed float and double values byte by byte. GCC does not merge that loop into one bswap, and C++23's std::byteswap accepts only integers. Overloads for float and double now copy the bits into a uint32_t/uint64_t, call detail::byteswap, and copy them back. This applies to every binary reader that uses get_number. In the CBOR reader, the single- and double-precision float cases call a non-inlined helper. Without it, GCC stopped inlining the stream adapter's get_character() into parse_cbor_internal, which made reading from a std::istream about 0.5 % slower. Signed-off-by: Niels Lohmann * CBOR reader: inline get_cbor_float() and always inline get() instead The never-inlined get_cbor_float() costs a call for every float read from input without bulk access: reading canada.json from CBOR through a FILE* became 5.6 % slower on Apple clang in the benchmark suite. The helper existed only because GCC 13 stopped inlining the stream adapter's get_character() into parse_cbor_internal. Always inlining binary_reader::get() fixes that as well. callgrind, all binary formats on buffer, std::istringstream and FILE* input (canada and citm_catalog): GCC 13 0.986-1.003, GCC 16 0.994-1.006, Clang 23 identical. Apple clang suite: FromBinaryFile/cbor/canada 0.939, geomean of the 53 binary reader rows 0.998. Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann --- .../nlohmann/detail/input/binary_reader.hpp | 45 +- single_include/nlohmann/json.hpp | 496 ++++++++++-------- 2 files changed, 300 insertions(+), 241 deletions(-) diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index 40df891bc..686b265a6 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -22,6 +22,7 @@ #include //byteswap #endif +#include #include #include #include @@ -1139,16 +1140,10 @@ class binary_reader return get_half_float(false); case 0xFA: // Single-Precision Float (four-byte IEEE 754) - { - float number{}; - return get_number(number) && emit_float(number); - } + return get_cbor_float(); case 0xFB: // Double-Precision Float (eight-byte IEEE 754) - { - double number{}; - return get_number(number) && emit_float(number); - } + return get_cbor_float(); default: // anything else (0xFF is handled inside the other types) { @@ -1157,6 +1152,14 @@ class binary_reader } } + /// read a single- or double-precision float + template + bool get_cbor_float() + { + NumberType number{}; + return get_number(number) && emit_float(number); + } + /*! @brief reports a nested indefinite-length CBOR string or byte array @param[in] type_name name of the rejected string type @@ -4176,7 +4179,12 @@ class binary_reader `char_traits::eof()` in that case. @return character read from the input + + @note always inlined: GCC otherwise stops inlining the stream adapter's + get_character() into parse_cbor_internal once that function grows + (+0.45 % instructions for CBOR from a std::istringstream with GCC 13) */ + JSON_HEDLEY_ALWAYS_INLINE char_int_type get() { ++chars_read; @@ -4222,6 +4230,27 @@ class binary_reader return current; } + /// reverse the byte order of a double with a single byteswap(): GCC does + /// not merge the byte-wise swap of the generic version into one + static void byte_swap(double& number) noexcept + { + static_assert(sizeof(double) == sizeof(std::uint64_t), "double must have 64 bits"); + std::uint64_t bits{}; + std::memcpy(&bits, &number, sizeof(bits)); + bits = byteswap(bits); + std::memcpy(&number, &bits, sizeof(bits)); + } + + /// reverse the byte order of a float with a single byteswap() + static void byte_swap(float& number) noexcept + { + static_assert(sizeof(float) == sizeof(std::uint32_t), "float must have 32 bits"); + std::uint32_t bits{}; + std::memcpy(&bits, &number, sizeof(bits)); + bits = byteswap(bits); + std::memcpy(&number, &bits, sizeof(bits)); + } + template static void byte_swap(NumberType& number) { diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index d16cbb3d4..fe5bf782b 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -8852,6 +8852,233 @@ NLOHMANN_JSON_NAMESPACE_END #include //byteswap #endif +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // uint64_t +#include // memcpy +#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_ARM64)) && (!defined(__SIZEOF_INT128__) || (!defined(__GNUC__) && !defined(__clang__))) + #include // __umulh, _umul128, _BitScanForward64, _BitScanReverse64 +#endif + +#ifdef _MSC_VER + #include // _byteswap_ushort, _byteswap_ulong, _byteswap_uint64 +#endif +#if defined(_MSC_VER) && !defined(__clang__) && (defined(_M_IX86) || defined(_M_ARM)) + #include // _BitScanForward, _BitScanReverse +#endif + +// #include +// JSON_HEDLEY_ALWAYS_INLINE, NLOHMANN_JSON_NAMESPACE_BEGIN + +#if defined(__cpp_lib_bit_cast) || defined(__cpp_lib_bitops) || defined(__cpp_lib_byteswap) + #include // bit_cast, byteswap, countl_zero, countr_zero +#endif + +// Portable bit-level helpers for the number and string scanners. They use +// compiler builtins or platform-specific intrinsics where available, the +// C++20 functions next, and plain C++ otherwise, so they work regardless +// of byte order. (The builtins and intrinsics come first: MSVC's +// std::countl_zero checks for LZCNT at run time unless AVX2 is enabled.) + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +/// number of leading zero bits of x (x != 0) +inline int count_leading_zeros(std::uint64_t x) noexcept +{ +#if defined(__GNUC__) || defined(__clang__) + return __builtin_clzll(x); +#elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_ARM64)) + unsigned long index = 0; // NOLINT(runtime/int): the type _BitScan*64 takes + _BitScanReverse64(&index, x); + return 63 - static_cast(index); +#elif defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_ARM)) + unsigned long index = 0; // NOLINT(runtime/int): the type _BitScan* takes + if (_BitScanReverse(&index, static_cast(x >> 32u))) // NOLINT(runtime/int) + { + return 31 - static_cast(index); + } + _BitScanReverse(&index, static_cast(x)); // NOLINT(runtime/int) + return 63 - static_cast(index); +#elif defined(__cpp_lib_bitops) + return std::countl_zero(x); +#else + int n = 0; + for (int shift = 32; shift != 0; shift >>= 1) + { + if ((x >> (64 - shift)) == 0) + { + n += shift; + x <<= shift; + } + } + return n; +#endif +} + +/// number of trailing zero bits of x (x != 0) +inline int count_trailing_zeros(std::uint64_t x) noexcept +{ +#if defined(__GNUC__) || defined(__clang__) + return __builtin_ctzll(x); +#elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_ARM64)) + unsigned long index = 0; // NOLINT(runtime/int): the type _BitScan*64 takes + _BitScanForward64(&index, x); + return static_cast(index); +#elif defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_ARM)) + unsigned long index = 0; // NOLINT(runtime/int): the type _BitScan* takes + if (_BitScanForward(&index, static_cast(x))) // NOLINT(runtime/int) + { + return static_cast(index); + } + _BitScanForward(&index, static_cast(x >> 32u)); // NOLINT(runtime/int) + return 32 + static_cast(index); +#elif defined(__cpp_lib_bitops) + return std::countr_zero(x); +#else + int n = 0; + for (int shift = 32; shift != 0; shift >>= 1) + { + if ((x << (64 - shift)) == 0) + { + n += shift; + x >>= shift; + } + } + return n; +#endif +} + +/// the object representation of @a from as a @a To (std::bit_cast in C++20) +template +JSON_HEDLEY_ALWAYS_INLINE To bit_cast(const From& from) noexcept +{ + static_assert(sizeof(To) == sizeof(From), "bit_cast needs types of the same size"); +#if defined(__cpp_lib_bit_cast) + return std::bit_cast(from); +#else + To to {}; + std::memcpy(&to, &from, sizeof(To)); + return to; +#endif +} + +/// x with its bytes in reverse order (std::byteswap in C++23); compilers lower +/// it to a single bswap/rev/movbe +inline std::uint16_t byteswap(std::uint16_t x) noexcept +{ +#if defined(__GNUC__) || defined(__clang__) + return __builtin_bswap16(x); +#elif defined(_MSC_VER) + return _byteswap_ushort(x); +#elif defined(__cpp_lib_byteswap) + return std::byteswap(x); +#else + return static_cast((x >> 8u) | (x << 8u)); +#endif +} + +/// x with its bytes in reverse order (std::byteswap in C++23) +inline std::uint32_t byteswap(std::uint32_t x) noexcept +{ +#if defined(__GNUC__) || defined(__clang__) + return __builtin_bswap32(x); +#elif defined(_MSC_VER) + return _byteswap_ulong(x); +#elif defined(__cpp_lib_byteswap) + return std::byteswap(x); +#else + return ((x & 0x000000FFu) << 24u) | ((x & 0x0000FF00u) << 8u) + | ((x & 0x00FF0000u) >> 8u) | ((x & 0xFF000000u) >> 24u); +#endif +} + +/// x with its bytes in reverse order (std::byteswap in C++23) +inline std::uint64_t byteswap(std::uint64_t x) noexcept +{ +#if defined(__GNUC__) || defined(__clang__) + return __builtin_bswap64(x); +#elif defined(_MSC_VER) + return _byteswap_uint64(x); +#elif defined(__cpp_lib_byteswap) + return std::byteswap(x); +#else + x = ((x & 0x00000000FFFFFFFFull) << 32u) | ((x & 0xFFFFFFFF00000000ull) >> 32u); + x = ((x & 0x0000FFFF0000FFFFull) << 16u) | ((x & 0xFFFF0000FFFF0000ull) >> 16u); + x = ((x & 0x00FF00FF00FF00FFull) << 8u) | ((x & 0xFF00FF00FF00FF00ull) >> 8u); + return x; +#endif +} + +/// the 128-bit product of two 64-bit numbers +struct uint128_parts +{ + std::uint64_t low; + std::uint64_t high; +}; + +inline uint128_parts full_multiplication(std::uint64_t a, std::uint64_t b) noexcept +{ +#if defined(__SIZEOF_INT128__) + __extension__ using uint128 = unsigned __int128; + const uint128 r = static_cast(a) * b; + return {static_cast(r), static_cast(r >> 64u)}; +#elif defined(_MSC_VER) && defined(_M_X64) + std::uint64_t high = 0; + const std::uint64_t low = _umul128(a, b, &high); + return {low, high}; +#elif defined(_MSC_VER) && defined(_M_ARM64) + return {a * b, __umulh(a, b)}; +#else + const std::uint64_t a_lo = a & 0xFFFFFFFFu; + const std::uint64_t a_hi = a >> 32u; + const std::uint64_t b_lo = b & 0xFFFFFFFFu; + const std::uint64_t b_hi = b >> 32u; + const std::uint64_t lo_lo = a_lo * b_lo; + const std::uint64_t hi_lo = a_hi * b_lo; + const std::uint64_t lo_hi = a_lo * b_hi; + const std::uint64_t hi_hi = a_hi * b_hi; + const std::uint64_t cross = (lo_lo >> 32u) + (hi_lo & 0xFFFFFFFFu) + lo_hi; + return {(cross << 32u) | (lo_lo & 0xFFFFFFFFu), (hi_lo >> 32u) + (cross >> 32u) + hi_hi}; +#endif +} + +/// eight bytes as a little-endian word (a single load on little-endian +/// targets; always inlined, as GCC otherwise calls it in the number loops) +JSON_HEDLEY_ALWAYS_INLINE std::uint64_t read_eight_bytes(const unsigned char* b) noexcept +{ +#if JSON_LITTLE_ENDIAN_TARGET + // the byte order already matches + std::uint64_t result = 0; + std::memcpy(&result, b, sizeof(result)); + return result; +#else + return static_cast(b[0]) | (static_cast(b[1]) << 8u) + | (static_cast(b[2]) << 16u) | (static_cast(b[3]) << 24u) + | (static_cast(b[4]) << 32u) | (static_cast(b[5]) << 40u) + | (static_cast(b[6]) << 48u) | (static_cast(b[7]) << 56u); +#endif +} + +/// eight bytes as a little-endian word +JSON_HEDLEY_ALWAYS_INLINE std::uint64_t read_eight_bytes(const char* p) noexcept +{ + return read_eight_bytes(reinterpret_cast(p)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) +} + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + // #include // #include @@ -10316,231 +10543,6 @@ NLOHMANN_JSON_NAMESPACE_END #include // vector // #include -// __ _____ _____ _____ -// __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.12.0 -// |_____|_____|_____|_|___| https://github.com/nlohmann/json -// -// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann -// SPDX-License-Identifier: MIT - - - -#include // uint64_t -#include // memcpy -#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_ARM64)) && (!defined(__SIZEOF_INT128__) || (!defined(__GNUC__) && !defined(__clang__))) - #include // __umulh, _umul128, _BitScanForward64, _BitScanReverse64 -#endif - -#ifdef _MSC_VER - #include // _byteswap_ushort, _byteswap_ulong, _byteswap_uint64 -#endif -#if defined(_MSC_VER) && !defined(__clang__) && (defined(_M_IX86) || defined(_M_ARM)) - #include // _BitScanForward, _BitScanReverse -#endif - -// #include -// JSON_HEDLEY_ALWAYS_INLINE, NLOHMANN_JSON_NAMESPACE_BEGIN - -#if defined(__cpp_lib_bit_cast) || defined(__cpp_lib_bitops) || defined(__cpp_lib_byteswap) - #include // bit_cast, byteswap, countl_zero, countr_zero -#endif - -// Portable bit-level helpers for the number and string scanners. They use -// compiler builtins or platform-specific intrinsics where available, the -// C++20 functions next, and plain C++ otherwise, so they work regardless -// of byte order. (The builtins and intrinsics come first: MSVC's -// std::countl_zero checks for LZCNT at run time unless AVX2 is enabled.) - -NLOHMANN_JSON_NAMESPACE_BEGIN -namespace detail -{ - -/// number of leading zero bits of x (x != 0) -inline int count_leading_zeros(std::uint64_t x) noexcept -{ -#if defined(__GNUC__) || defined(__clang__) - return __builtin_clzll(x); -#elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_ARM64)) - unsigned long index = 0; // NOLINT(runtime/int): the type _BitScan*64 takes - _BitScanReverse64(&index, x); - return 63 - static_cast(index); -#elif defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_ARM)) - unsigned long index = 0; // NOLINT(runtime/int): the type _BitScan* takes - if (_BitScanReverse(&index, static_cast(x >> 32u))) // NOLINT(runtime/int) - { - return 31 - static_cast(index); - } - _BitScanReverse(&index, static_cast(x)); // NOLINT(runtime/int) - return 63 - static_cast(index); -#elif defined(__cpp_lib_bitops) - return std::countl_zero(x); -#else - int n = 0; - for (int shift = 32; shift != 0; shift >>= 1) - { - if ((x >> (64 - shift)) == 0) - { - n += shift; - x <<= shift; - } - } - return n; -#endif -} - -/// number of trailing zero bits of x (x != 0) -inline int count_trailing_zeros(std::uint64_t x) noexcept -{ -#if defined(__GNUC__) || defined(__clang__) - return __builtin_ctzll(x); -#elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_ARM64)) - unsigned long index = 0; // NOLINT(runtime/int): the type _BitScan*64 takes - _BitScanForward64(&index, x); - return static_cast(index); -#elif defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_ARM)) - unsigned long index = 0; // NOLINT(runtime/int): the type _BitScan* takes - if (_BitScanForward(&index, static_cast(x))) // NOLINT(runtime/int) - { - return static_cast(index); - } - _BitScanForward(&index, static_cast(x >> 32u)); // NOLINT(runtime/int) - return 32 + static_cast(index); -#elif defined(__cpp_lib_bitops) - return std::countr_zero(x); -#else - int n = 0; - for (int shift = 32; shift != 0; shift >>= 1) - { - if ((x << (64 - shift)) == 0) - { - n += shift; - x >>= shift; - } - } - return n; -#endif -} - -/// the object representation of @a from as a @a To (std::bit_cast in C++20) -template -JSON_HEDLEY_ALWAYS_INLINE To bit_cast(const From& from) noexcept -{ - static_assert(sizeof(To) == sizeof(From), "bit_cast needs types of the same size"); -#if defined(__cpp_lib_bit_cast) - return std::bit_cast(from); -#else - To to {}; - std::memcpy(&to, &from, sizeof(To)); - return to; -#endif -} - -/// x with its bytes in reverse order (std::byteswap in C++23); compilers lower -/// it to a single bswap/rev/movbe -inline std::uint16_t byteswap(std::uint16_t x) noexcept -{ -#if defined(__GNUC__) || defined(__clang__) - return __builtin_bswap16(x); -#elif defined(_MSC_VER) - return _byteswap_ushort(x); -#elif defined(__cpp_lib_byteswap) - return std::byteswap(x); -#else - return static_cast((x >> 8u) | (x << 8u)); -#endif -} - -/// x with its bytes in reverse order (std::byteswap in C++23) -inline std::uint32_t byteswap(std::uint32_t x) noexcept -{ -#if defined(__GNUC__) || defined(__clang__) - return __builtin_bswap32(x); -#elif defined(_MSC_VER) - return _byteswap_ulong(x); -#elif defined(__cpp_lib_byteswap) - return std::byteswap(x); -#else - return ((x & 0x000000FFu) << 24u) | ((x & 0x0000FF00u) << 8u) - | ((x & 0x00FF0000u) >> 8u) | ((x & 0xFF000000u) >> 24u); -#endif -} - -/// x with its bytes in reverse order (std::byteswap in C++23) -inline std::uint64_t byteswap(std::uint64_t x) noexcept -{ -#if defined(__GNUC__) || defined(__clang__) - return __builtin_bswap64(x); -#elif defined(_MSC_VER) - return _byteswap_uint64(x); -#elif defined(__cpp_lib_byteswap) - return std::byteswap(x); -#else - x = ((x & 0x00000000FFFFFFFFull) << 32u) | ((x & 0xFFFFFFFF00000000ull) >> 32u); - x = ((x & 0x0000FFFF0000FFFFull) << 16u) | ((x & 0xFFFF0000FFFF0000ull) >> 16u); - x = ((x & 0x00FF00FF00FF00FFull) << 8u) | ((x & 0xFF00FF00FF00FF00ull) >> 8u); - return x; -#endif -} - -/// the 128-bit product of two 64-bit numbers -struct uint128_parts -{ - std::uint64_t low; - std::uint64_t high; -}; - -inline uint128_parts full_multiplication(std::uint64_t a, std::uint64_t b) noexcept -{ -#if defined(__SIZEOF_INT128__) - __extension__ using uint128 = unsigned __int128; - const uint128 r = static_cast(a) * b; - return {static_cast(r), static_cast(r >> 64u)}; -#elif defined(_MSC_VER) && defined(_M_X64) - std::uint64_t high = 0; - const std::uint64_t low = _umul128(a, b, &high); - return {low, high}; -#elif defined(_MSC_VER) && defined(_M_ARM64) - return {a * b, __umulh(a, b)}; -#else - const std::uint64_t a_lo = a & 0xFFFFFFFFu; - const std::uint64_t a_hi = a >> 32u; - const std::uint64_t b_lo = b & 0xFFFFFFFFu; - const std::uint64_t b_hi = b >> 32u; - const std::uint64_t lo_lo = a_lo * b_lo; - const std::uint64_t hi_lo = a_hi * b_lo; - const std::uint64_t lo_hi = a_lo * b_hi; - const std::uint64_t hi_hi = a_hi * b_hi; - const std::uint64_t cross = (lo_lo >> 32u) + (hi_lo & 0xFFFFFFFFu) + lo_hi; - return {(cross << 32u) | (lo_lo & 0xFFFFFFFFu), (hi_lo >> 32u) + (cross >> 32u) + hi_hi}; -#endif -} - -/// eight bytes as a little-endian word (a single load on little-endian -/// targets; always inlined, as GCC otherwise calls it in the number loops) -JSON_HEDLEY_ALWAYS_INLINE std::uint64_t read_eight_bytes(const unsigned char* b) noexcept -{ -#if JSON_LITTLE_ENDIAN_TARGET - // the byte order already matches - std::uint64_t result = 0; - std::memcpy(&result, b, sizeof(result)); - return result; -#else - return static_cast(b[0]) | (static_cast(b[1]) << 8u) - | (static_cast(b[2]) << 16u) | (static_cast(b[3]) << 24u) - | (static_cast(b[4]) << 32u) | (static_cast(b[5]) << 40u) - | (static_cast(b[6]) << 48u) | (static_cast(b[7]) << 56u); -#endif -} - -/// eight bytes as a little-endian word -JSON_HEDLEY_ALWAYS_INLINE std::uint64_t read_eight_bytes(const char* p) noexcept -{ - return read_eight_bytes(reinterpret_cast(p)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) -} - -} // namespace detail -NLOHMANN_JSON_NAMESPACE_END // #include @@ -18943,16 +18945,10 @@ class binary_reader return get_half_float(false); case 0xFA: // Single-Precision Float (four-byte IEEE 754) - { - float number{}; - return get_number(number) && emit_float(number); - } + return get_cbor_float(); case 0xFB: // Double-Precision Float (eight-byte IEEE 754) - { - double number{}; - return get_number(number) && emit_float(number); - } + return get_cbor_float(); default: // anything else (0xFF is handled inside the other types) { @@ -18961,6 +18957,14 @@ class binary_reader } } + /// read a single- or double-precision float + template + bool get_cbor_float() + { + NumberType number{}; + return get_number(number) && emit_float(number); + } + /*! @brief reports a nested indefinite-length CBOR string or byte array @param[in] type_name name of the rejected string type @@ -21980,7 +21984,12 @@ class binary_reader `char_traits::eof()` in that case. @return character read from the input + + @note always inlined: GCC otherwise stops inlining the stream adapter's + get_character() into parse_cbor_internal once that function grows + (+0.45 % instructions for CBOR from a std::istringstream with GCC 13) */ + JSON_HEDLEY_ALWAYS_INLINE char_int_type get() { ++chars_read; @@ -22026,6 +22035,27 @@ class binary_reader return current; } + /// reverse the byte order of a double with a single byteswap(): GCC does + /// not merge the byte-wise swap of the generic version into one + static void byte_swap(double& number) noexcept + { + static_assert(sizeof(double) == sizeof(std::uint64_t), "double must have 64 bits"); + std::uint64_t bits{}; + std::memcpy(&bits, &number, sizeof(bits)); + bits = byteswap(bits); + std::memcpy(&number, &bits, sizeof(bits)); + } + + /// reverse the byte order of a float with a single byteswap() + static void byte_swap(float& number) noexcept + { + static_assert(sizeof(float) == sizeof(std::uint32_t), "float must have 32 bits"); + std::uint32_t bits{}; + std::memcpy(&bits, &number, sizeof(bits)); + bits = byteswap(bits); + std::memcpy(&number, &bits, sizeof(bits)); + } + template static void byte_swap(NumberType& number) { From edf8d1760129d82162d0d1ca7e8fb2d0c08db638 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 1 Nov 2026 17:22:32 +0100 Subject: [PATCH 13/54] MessagePack reader: read runs of scalar array elements in one loop (#5980) * MessagePack reader: read runs of scalar array elements in one loop As for CBOR: when an array opens and the input is contiguous, read the integers, floats, booleans, and nils that follow in a tight loop instead of going through parse_msgpack_internal and parse_msgpack_value for every element. The loop stops at any other kind of value and when fewer than 9 bytes are left; those values are read by the general path as before. Non-contiguous inputs are not affected. Signed-off-by: Niels Lohmann * MessagePack reader: enter the scalar run only when a scalar follows As for CBOR: enter_msgpack_array tests the first byte and the remaining length inline and calls the out-of-line run only for a scalar, so arrays of maps or strings no longer pay for a call that reads nothing; the run's loop condition moves to the end (do-while). Signed-off-by: Niels Lohmann * MessagePack reader: mark the initial accepted in the scalar run as a false-positive dead store ci_infer reports DEAD_STORE for 'bool accepted = true;' in the MessagePack scalar run, as in the CBOR one, because every case assigns accepted before it is read. Signed-off-by: Niels Lohmann * MessagePack tests: assign the parsed value instead of casting it to void GCC (ci_test_gcc) reports -Werror=unused-result for static_cast(json::from_msgpack(...)): a cast to void does not silence warn_unused_result on GCC. Assign to a json, as the other tests in the file do. Signed-off-by: Niels Lohmann * MessagePack reader: drop the scalar pre-check before the run enter_msgpack_array now calls the run directly, and parse_msgpack_scalar_run tests the remaining count and length at the top of a while loop instead of relying on msgpack_scalar_follows(); its own switch already leaves on any value that is not a scalar. Instructions per repetition, new/old: SAX canada 0.946 (clang) and 0.973 (GCC 13), DOM read canada 0.991 / 0.992; no row more than 0.3 % higher. Signed-off-by: Niels Lohmann * MessagePack reader: say why the initial accepted in the scalar run stays As for CBOR: Infer's DEAD_STORE report is right, not a false positive. The initializer is kept for clang-tidy's cppcoreguidelines-init-variables. Signed-off-by: Niels Lohmann * MessagePack reader: give the scalar run the bulk tag instead of a wrapper As for CBOR: parse_msgpack_scalar_run takes the bulk-scan tag itself and is the function kept out of line, so parse_msgpack_array_start goes. Signed-off-by: Niels Lohmann * MessagePack reader: start the scalar run's accepted at false As for CBOR: Infer ignores dead stores of 0, so the suppression goes. Signed-off-by: Niels Lohmann * MessagePack reader: let the scalar run find its array itself As for CBOR: parse_msgpack_scalar_run reads container_stack.back() in its bulk version, so the caller no longer evaluates it for input that is not contiguous. Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann --- .../nlohmann/detail/input/binary_reader.hpp | 137 +++++++++++++++++- single_include/nlohmann/json.hpp | 137 +++++++++++++++++- tests/src/unit-msgpack.cpp | 99 +++++++++++++ 3 files changed, 367 insertions(+), 6 deletions(-) diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index 686b265a6..0c345fea0 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -2130,7 +2130,7 @@ class binary_reader case 0x9D: case 0x9E: case 0x9F: - return enter_array(conditional_static_cast(static_cast(current) & 0x0Fu)); + return enter_msgpack_array(conditional_static_cast(static_cast(current) & 0x0Fu)); // fixstr case 0xA0: @@ -2261,13 +2261,13 @@ class binary_reader case 0xDC: // array 16 { std::uint16_t len{}; - return get_number(len) && enter_array(static_cast(len)); + return get_number(len) && enter_msgpack_array(static_cast(len)); } case 0xDD: // array 32 { std::uint32_t len{}; - return get_number(len) && enter_array(conditional_static_cast(len)); + return get_number(len) && enter_msgpack_array(conditional_static_cast(len)); } case 0xDE: // map 16 @@ -2609,6 +2609,137 @@ class binary_reader } } + /*! + @brief read the integers, floats, booleans, and nils that follow in a + MessagePack array, without the per-element bookkeeping of + @ref parse_msgpack_internal + + Stops at the end of the array, at any other kind of value, and when fewer + than 9 bytes (the longest of these values) are left in the input, leaving + those to @ref parse_msgpack_value, which reads them the same way. + + Out of line, so that @ref parse_msgpack_value stays small enough to be inlined. + + The array is the one on top of @ref container_stack; its count of + elements not yet read is updated. + + @return whether the SAX parser accepted the values + */ + JSON_HEDLEY_NEVER_INLINE + bool parse_msgpack_scalar_run(std::true_type /*bulk*/) + { + // nothing pushes onto the stack while the run reads scalars + std::size_t& remaining = container_stack.back().remaining; + while (remaining != 0 && ia.bulk_remaining() >= 9) + { + const auto* const data = reinterpret_cast(ia.bulk_data()); + const unsigned char head = data[0]; + std::size_t size = 0; + bool accepted = false; + switch (head) + { + case 0xC0: // nil + size = 1; + accepted = sax->null(); + break; + case 0xC2: // false + size = 1; + accepted = sax->boolean(false); + break; + case 0xC3: // true + size = 1; + accepted = sax->boolean(true); + break; + case 0xCA: // float 32 + size = 5; + accepted = emit_float(load_big_endian(data + 1)); + break; + case 0xCB: // float 64 + size = 9; + accepted = emit_float(load_big_endian(data + 1)); + break; + case 0xCC: // uint 8 + size = 2; + accepted = emit_unsigned(data[1]); + break; + case 0xCD: // uint 16 + size = 3; + accepted = emit_unsigned(load_big_endian(data + 1)); + break; + case 0xCE: // uint 32 + size = 5; + accepted = emit_unsigned(load_big_endian(data + 1)); + break; + case 0xCF: // uint 64 + size = 9; + accepted = emit_unsigned(load_big_endian(data + 1)); + break; + case 0xD0: // int 8 + size = 2; + accepted = emit_signed(static_cast(data[1])); + break; + case 0xD1: // int 16 + size = 3; + accepted = emit_signed(load_big_endian(data + 1)); + break; + case 0xD2: // int 32 + size = 5; + accepted = emit_signed(load_big_endian(data + 1)); + break; + case 0xD3: // int 64 + size = 9; + accepted = emit_signed(load_big_endian(data + 1)); + break; + default: + if (head <= 0x7F) + { + // positive fixint + size = 1; + accepted = sax->number_unsigned(static_cast(head)); + } + else if (head >= 0xE0) + { + // negative fixint + size = 1; + accepted = sax->number_integer(static_cast(head)); + } + else + { + // any other value is read by parse_msgpack_value + return true; + } + break; + } + // as if read by get() and get_number(): current is the last byte + ia.bulk_skip(size); + chars_read += size; + current = data[size - 1]; + --remaining; + if (JSON_HEDLEY_UNLIKELY(!accepted)) + { + return false; + } + } + return true; + } + + /*! + @brief open a MessagePack array and read the scalars it begins with (see + @ref parse_msgpack_scalar_run) + @param[in] len the number of elements + @return whether the SAX parser accepted the array and the values read + */ + bool enter_msgpack_array(const std::size_t len) + { + return enter_array(len) && (len == 0 || parse_msgpack_scalar_run(std::integral_constant {})); + } + + /// input that is not contiguous: every value is read by parse_msgpack_value + bool parse_msgpack_scalar_run(std::false_type /*bulk*/) const noexcept + { + return true; + } + /*! @brief read a MessagePack value and everything nested inside it diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index fe5bf782b..d440efd8f 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -19935,7 +19935,7 @@ class binary_reader case 0x9D: case 0x9E: case 0x9F: - return enter_array(conditional_static_cast(static_cast(current) & 0x0Fu)); + return enter_msgpack_array(conditional_static_cast(static_cast(current) & 0x0Fu)); // fixstr case 0xA0: @@ -20066,13 +20066,13 @@ class binary_reader case 0xDC: // array 16 { std::uint16_t len{}; - return get_number(len) && enter_array(static_cast(len)); + return get_number(len) && enter_msgpack_array(static_cast(len)); } case 0xDD: // array 32 { std::uint32_t len{}; - return get_number(len) && enter_array(conditional_static_cast(len)); + return get_number(len) && enter_msgpack_array(conditional_static_cast(len)); } case 0xDE: // map 16 @@ -20414,6 +20414,137 @@ class binary_reader } } + /*! + @brief read the integers, floats, booleans, and nils that follow in a + MessagePack array, without the per-element bookkeeping of + @ref parse_msgpack_internal + + Stops at the end of the array, at any other kind of value, and when fewer + than 9 bytes (the longest of these values) are left in the input, leaving + those to @ref parse_msgpack_value, which reads them the same way. + + Out of line, so that @ref parse_msgpack_value stays small enough to be inlined. + + The array is the one on top of @ref container_stack; its count of + elements not yet read is updated. + + @return whether the SAX parser accepted the values + */ + JSON_HEDLEY_NEVER_INLINE + bool parse_msgpack_scalar_run(std::true_type /*bulk*/) + { + // nothing pushes onto the stack while the run reads scalars + std::size_t& remaining = container_stack.back().remaining; + while (remaining != 0 && ia.bulk_remaining() >= 9) + { + const auto* const data = reinterpret_cast(ia.bulk_data()); + const unsigned char head = data[0]; + std::size_t size = 0; + bool accepted = false; + switch (head) + { + case 0xC0: // nil + size = 1; + accepted = sax->null(); + break; + case 0xC2: // false + size = 1; + accepted = sax->boolean(false); + break; + case 0xC3: // true + size = 1; + accepted = sax->boolean(true); + break; + case 0xCA: // float 32 + size = 5; + accepted = emit_float(load_big_endian(data + 1)); + break; + case 0xCB: // float 64 + size = 9; + accepted = emit_float(load_big_endian(data + 1)); + break; + case 0xCC: // uint 8 + size = 2; + accepted = emit_unsigned(data[1]); + break; + case 0xCD: // uint 16 + size = 3; + accepted = emit_unsigned(load_big_endian(data + 1)); + break; + case 0xCE: // uint 32 + size = 5; + accepted = emit_unsigned(load_big_endian(data + 1)); + break; + case 0xCF: // uint 64 + size = 9; + accepted = emit_unsigned(load_big_endian(data + 1)); + break; + case 0xD0: // int 8 + size = 2; + accepted = emit_signed(static_cast(data[1])); + break; + case 0xD1: // int 16 + size = 3; + accepted = emit_signed(load_big_endian(data + 1)); + break; + case 0xD2: // int 32 + size = 5; + accepted = emit_signed(load_big_endian(data + 1)); + break; + case 0xD3: // int 64 + size = 9; + accepted = emit_signed(load_big_endian(data + 1)); + break; + default: + if (head <= 0x7F) + { + // positive fixint + size = 1; + accepted = sax->number_unsigned(static_cast(head)); + } + else if (head >= 0xE0) + { + // negative fixint + size = 1; + accepted = sax->number_integer(static_cast(head)); + } + else + { + // any other value is read by parse_msgpack_value + return true; + } + break; + } + // as if read by get() and get_number(): current is the last byte + ia.bulk_skip(size); + chars_read += size; + current = data[size - 1]; + --remaining; + if (JSON_HEDLEY_UNLIKELY(!accepted)) + { + return false; + } + } + return true; + } + + /*! + @brief open a MessagePack array and read the scalars it begins with (see + @ref parse_msgpack_scalar_run) + @param[in] len the number of elements + @return whether the SAX parser accepted the array and the values read + */ + bool enter_msgpack_array(const std::size_t len) + { + return enter_array(len) && (len == 0 || parse_msgpack_scalar_run(std::integral_constant {})); + } + + /// input that is not contiguous: every value is read by parse_msgpack_value + bool parse_msgpack_scalar_run(std::false_type /*bulk*/) const noexcept + { + return true; + } + /*! @brief read a MessagePack value and everything nested inside it diff --git a/tests/src/unit-msgpack.cpp b/tests/src/unit-msgpack.cpp index 2890a240c..9b5891fdd 100644 --- a/tests/src/unit-msgpack.cpp +++ b/tests/src/unit-msgpack.cpp @@ -27,12 +27,15 @@ using nlohmann::json; #include #include #include +#include #include #include "make_test_data_available.hpp" #include "round_trip_corpus.hpp" #include "test_utils.hpp" #include "sax_countdown.hpp" +#include "sax_event_loggers.hpp" using utils::SaxCountdown; +using utils::SaxEventLogger; TEST_CASE("MessagePack") @@ -2212,3 +2215,99 @@ TEST_CASE("MessagePack large strings and binaries (chunked reader)") } } } + +TEST_CASE("MessagePack arrays of scalars from contiguous and non-contiguous input") +{ + // contiguous input reads the integers, floats, booleans, and nils an + // array begins with in one loop, other input value by value; both must + // give the same events, value, or error for the complete document and for + // every truncation of it, and stop at every event a SAX handler rejects + const std::vector msgpack = + { + 0x83, // map(3) + 0xA1, 'a', // "a" + 0xDC, 0x00, 0x16, // array 16 (22) + 0x00, 0x7F, 0xCC, 0xFF, 0xCD, 0x01, 0x00, // 0, 127, 255, 256 + 0xCE, 0x00, 0x01, 0x00, 0x00, // 65536 + 0xCF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 18446744073709551615 + 0xFF, 0xE0, 0xD0, 0x80, 0xD1, 0xFF, 0x00, // -1, -32, -128, -256 + 0xD2, 0xFF, 0xFE, 0xFF, 0xFF, // -65537 + 0xD3, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // -9223372036854775808 + 0xC2, 0xC3, 0xC0, // false, true, nil + 0xCA, 0x3F, 0xC0, 0x00, 0x00, // 1.5 (float 32) + 0xCB, 0xBF, 0xB9, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9A, // -0.1 + 0xA1, 'x', // "x" (ends the loop) + 0x01, // 1 + 0xCB, 0x40, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 2.5 + 0xD9, 0x01, 'y', // "y" (str 8) + 0xC0, // nil + 0xA1, 'b', // "b" + 0x93, // fixarray(3) + 0x92, 0xCB, 0x3F, 0xF4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // [1.25, + 0xCB, 0xC0, 0x0E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // -3.75] + 0x92, 0xCB, 0x3F, 0xF4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // [1.25, + 0xCB, 0xC0, 0x0E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // -3.75] + 0x90, // [] + 0xA1, 'c', // "c" + 0x94, 0x80, 0x01, 0xC4, 0x01, 0x05, 0x91, 0x03 // [{}, 1, bin 8, [3]] + }; + + const json j = json::from_msgpack(msgpack); + CHECK(j["a"].size() == 22); + CHECK(j["a"][5] == 18446744073709551615u); + CHECK(j["a"][7] == -32); + CHECK(j["a"][11] == (std::numeric_limits::min)()); + CHECK(j["a"][15] == 1.5); + CHECK(j["a"][16] == -0.1); + CHECK(j["a"][17] == "x"); + CHECK(j["b"] == json::parse("[[1.25, -3.75], [1.25, -3.75], []]")); + CHECK(j["c"][2].is_binary()); + + for (std::size_t length = 0; length <= msgpack.size(); ++length) + { + CAPTURE(length) + const std::vector contiguous(msgpack.begin(), msgpack.begin() + static_cast(length)); + const std::list bytewise(contiguous.begin(), contiguous.end()); + + CHECK(json::from_msgpack(contiguous, true, false).dump() == json::from_msgpack(bytewise.begin(), bytewise.end(), true, false).dump()); +#if !defined(JSON_NOEXCEPTION) + std::string contiguous_error; + std::string bytewise_error; + try + { + json _; + _ = json::from_msgpack(contiguous); + } + catch (const json::parse_error& e) + { + contiguous_error = e.what(); + } + try + { + json _; + _ = json::from_msgpack(bytewise.begin(), bytewise.end()); + } + catch (const json::parse_error& e) + { + bytewise_error = e.what(); + } + CHECK(contiguous_error == bytewise_error); +#endif + + SaxEventLogger from_contiguous; + SaxEventLogger from_bytewise; + CHECK(json::sax_parse(contiguous, &from_contiguous, json::input_format_t::msgpack) + == json::sax_parse(bytewise.begin(), bytewise.end(), &from_bytewise, json::input_format_t::msgpack)); + CHECK(from_contiguous.events == from_bytewise.events); + } + + for (int events = 0; events <= 60; ++events) + { + CAPTURE(events) + const std::list bytewise(msgpack.begin(), msgpack.end()); + SaxCountdown from_contiguous(events); + SaxCountdown from_bytewise(events); + CHECK(json::sax_parse(msgpack, &from_contiguous, json::input_format_t::msgpack) + == json::sax_parse(bytewise.begin(), bytewise.end(), &from_bytewise, json::input_format_t::msgpack)); + } +} From fb515ddc34565b3d78b56be2f43919846d45fa24 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 1 Nov 2026 17:22:33 +0100 Subject: [PATCH 14/54] UBJSON/BJData readers: read optimized arrays of numbers in one loop (#5981) * UBJSON/BJData readers: read optimized arrays of numbers in one loop An array with a type marker for a fixed-size number ($U, $i, $I, $l, $L, $d, $D, and BJData's $u, $m, $M), including the data of a BJData ndarray, gives its elements no markers. When the input is contiguous, read the elements that are in the input in one loop - load, byte swap for the format's byte order, emit - instead of one trip through parse_ubjson_internal and get_ubjson_value per element. Elements the input ends before, other types, and non-contiguous inputs are read as before. Signed-off-by: Niels Lohmann * UBJSON/BJData tests: assign the parsed value instead of casting it to void GCC reports -Werror=unused-result for static_cast(json::from_ubjson(...)) and json::from_bjdata(...): a cast to void does not silence warn_unused_result on GCC. Assign to a json, as the other tests do. Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann --- .../nlohmann/detail/input/binary_reader.hpp | 114 +++++++++++++++++- single_include/nlohmann/json.hpp | 114 +++++++++++++++++- tests/src/unit-bjdata.cpp | 77 ++++++++++++ tests/src/unit-ubjson.cpp | 72 +++++++++++ 4 files changed, 373 insertions(+), 4 deletions(-) diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index 0c345fea0..190edb112 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -3545,6 +3545,107 @@ class binary_reader return unexpected_byte("invalid byte", "value"); } + /*! + @brief read the elements of an optimized array of numbers (type marker + @a type) that are in the input, in one loop, without the + per-element bookkeeping of @ref parse_ubjson_internal + + Values are emitted as @ref get_ubjson_value emits them. Types other than + the fixed-size integers and floats are left to @ref get_ubjson_value, as + are the elements the input ends before. + + @param[in,out] remaining the number of elements not yet read + @param[in] type the type marker of the elements + @return whether the SAX parser accepted the values + */ + JSON_HEDLEY_NEVER_INLINE + bool parse_ubjson_typed_run(std::size_t& remaining, const char_int_type type, std::true_type /*bulk*/) + { + const bool bjdata = input_format == input_format_t::bjdata; + switch (type) + { + case 'U': + return parse_ubjson_typed_run(remaining); + case 'i': + return parse_ubjson_typed_run(remaining); + case 'I': + return parse_ubjson_typed_run(remaining); + case 'l': + return parse_ubjson_typed_run(remaining); + case 'L': + return parse_ubjson_typed_run(remaining); + case 'u': + return !bjdata || parse_ubjson_typed_run(remaining); + case 'm': + return !bjdata || parse_ubjson_typed_run(remaining); + case 'M': + return !bjdata || parse_ubjson_typed_run(remaining); + case 'd': + return parse_ubjson_typed_run(remaining); + case 'D': + return parse_ubjson_typed_run(remaining); + default: + return true; + } + } + + /// input that is not contiguous: every element is read by get_ubjson_value + bool parse_ubjson_typed_run(std::size_t& /*remaining*/, const char_int_type /*type*/, std::false_type /*bulk*/) const noexcept + { + return true; + } + + template + bool parse_ubjson_typed_run(std::size_t& remaining) + { + constexpr std::size_t size = sizeof(NumberType); + const std::size_t count = (std::min)(remaining, ia.bulk_remaining() / size); + if (count == 0) + { + return true; + } + const auto* const data = reinterpret_cast(ia.bulk_data()); + // UBJSON stores numbers big-endian, BJData little-endian + const bool swap = is_little_endian != (input_format == input_format_t::bjdata); + std::size_t read = 0; + bool accepted = true; + while (read < count && accepted) + { + NumberType number{}; + std::memcpy(&number, data + (read * size), size); + if (swap) + { + byte_swap(number); + } + ++read; + accepted = emit_ubjson_number(number); + } + // as if read by get_number(): current is the last byte read + ia.bulk_skip(read * size); + chars_read += read * size; + current = data[(read * size) - 1]; + remaining -= read; + return accepted; + } + + template::value, int> = 0> + bool emit_ubjson_number(const NumberType number) + { + return emit_float(number); + } + + template < typename NumberType, enable_if_t < std::is_integral::value&& std::is_signed::value, int > = 0 > + bool emit_ubjson_number(const NumberType number) + { + return emit_signed(number); + } + + template < typename NumberType, enable_if_t < std::is_integral::value&& !std::is_signed::value, int > = 0 > + bool emit_ubjson_number(const NumberType number) + { + return emit_unsigned(number); + } + /*! @return whether array creation completed */ @@ -3580,7 +3681,15 @@ class binary_reader return false; } - for (std::size_t i = 0; i < size_and_type.first; ++i) + // the numbers that are in the input, in one loop; the rest (if + // the input ends early) one by one below + std::size_t remaining = size_and_type.first; + if (!nd_is_char && JSON_HEDLEY_UNLIKELY(!parse_ubjson_typed_run(remaining, size_and_type.second, std::integral_constant {}))) + { + return false; + } + + for (std::size_t i = size_and_type.first - remaining; i < size_and_type.first; ++i) { if (nd_is_char) { @@ -3640,7 +3749,8 @@ class binary_reader container_stack.back().remaining = 0; } - return true; + return size_and_type.second == 0 + || parse_ubjson_typed_run(container_stack.back().remaining, size_and_type.second, std::integral_constant {}); } return enter_array(detail::unknown_size()); diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index d440efd8f..84720a816 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -21350,6 +21350,107 @@ class binary_reader return unexpected_byte("invalid byte", "value"); } + /*! + @brief read the elements of an optimized array of numbers (type marker + @a type) that are in the input, in one loop, without the + per-element bookkeeping of @ref parse_ubjson_internal + + Values are emitted as @ref get_ubjson_value emits them. Types other than + the fixed-size integers and floats are left to @ref get_ubjson_value, as + are the elements the input ends before. + + @param[in,out] remaining the number of elements not yet read + @param[in] type the type marker of the elements + @return whether the SAX parser accepted the values + */ + JSON_HEDLEY_NEVER_INLINE + bool parse_ubjson_typed_run(std::size_t& remaining, const char_int_type type, std::true_type /*bulk*/) + { + const bool bjdata = input_format == input_format_t::bjdata; + switch (type) + { + case 'U': + return parse_ubjson_typed_run(remaining); + case 'i': + return parse_ubjson_typed_run(remaining); + case 'I': + return parse_ubjson_typed_run(remaining); + case 'l': + return parse_ubjson_typed_run(remaining); + case 'L': + return parse_ubjson_typed_run(remaining); + case 'u': + return !bjdata || parse_ubjson_typed_run(remaining); + case 'm': + return !bjdata || parse_ubjson_typed_run(remaining); + case 'M': + return !bjdata || parse_ubjson_typed_run(remaining); + case 'd': + return parse_ubjson_typed_run(remaining); + case 'D': + return parse_ubjson_typed_run(remaining); + default: + return true; + } + } + + /// input that is not contiguous: every element is read by get_ubjson_value + bool parse_ubjson_typed_run(std::size_t& /*remaining*/, const char_int_type /*type*/, std::false_type /*bulk*/) const noexcept + { + return true; + } + + template + bool parse_ubjson_typed_run(std::size_t& remaining) + { + constexpr std::size_t size = sizeof(NumberType); + const std::size_t count = (std::min)(remaining, ia.bulk_remaining() / size); + if (count == 0) + { + return true; + } + const auto* const data = reinterpret_cast(ia.bulk_data()); + // UBJSON stores numbers big-endian, BJData little-endian + const bool swap = is_little_endian != (input_format == input_format_t::bjdata); + std::size_t read = 0; + bool accepted = true; + while (read < count && accepted) + { + NumberType number{}; + std::memcpy(&number, data + (read * size), size); + if (swap) + { + byte_swap(number); + } + ++read; + accepted = emit_ubjson_number(number); + } + // as if read by get_number(): current is the last byte read + ia.bulk_skip(read * size); + chars_read += read * size; + current = data[(read * size) - 1]; + remaining -= read; + return accepted; + } + + template::value, int> = 0> + bool emit_ubjson_number(const NumberType number) + { + return emit_float(number); + } + + template < typename NumberType, enable_if_t < std::is_integral::value&& std::is_signed::value, int > = 0 > + bool emit_ubjson_number(const NumberType number) + { + return emit_signed(number); + } + + template < typename NumberType, enable_if_t < std::is_integral::value&& !std::is_signed::value, int > = 0 > + bool emit_ubjson_number(const NumberType number) + { + return emit_unsigned(number); + } + /*! @return whether array creation completed */ @@ -21385,7 +21486,15 @@ class binary_reader return false; } - for (std::size_t i = 0; i < size_and_type.first; ++i) + // the numbers that are in the input, in one loop; the rest (if + // the input ends early) one by one below + std::size_t remaining = size_and_type.first; + if (!nd_is_char && JSON_HEDLEY_UNLIKELY(!parse_ubjson_typed_run(remaining, size_and_type.second, std::integral_constant {}))) + { + return false; + } + + for (std::size_t i = size_and_type.first - remaining; i < size_and_type.first; ++i) { if (nd_is_char) { @@ -21445,7 +21554,8 @@ class binary_reader container_stack.back().remaining = 0; } - return true; + return size_and_type.second == 0 + || parse_ubjson_typed_run(container_stack.back().remaining, size_and_type.second, std::integral_constant {}); } return enter_array(detail::unknown_size()); diff --git a/tests/src/unit-bjdata.cpp b/tests/src/unit-bjdata.cpp index e4dffb6d0..04ee3acc4 100644 --- a/tests/src/unit-bjdata.cpp +++ b/tests/src/unit-bjdata.cpp @@ -26,11 +26,14 @@ using nlohmann::ordered_json; #include #include #include +#include #include #include "make_test_data_available.hpp" #include "test_utils.hpp" #include "sax_countdown.hpp" +#include "sax_event_loggers.hpp" using utils::SaxCountdown; +using utils::SaxEventLogger; // trait_test_arg and the "value_in_range_of trait" TEST_CASE_TEMPLATE_DEFINE @@ -4746,3 +4749,77 @@ TEST_CASE("BJData optimized container type marker is validated for count 0") CHECK(json::from_bjdata(vb) == json::binary({})); } } + +TEST_CASE("BJData optimized arrays from contiguous and non-contiguous input") +{ + // contiguous input reads the numbers of an array with a type marker in + // one loop, other input number by number; both must give the same events, + // value, or error for the complete document and for every truncation of + // it, and stop at every event a SAX handler rejects + json document = + { + {"u8", {0, 200, 255}}, + {"i8", {-5, 100, -128}}, + {"i16", {-30000, 1000, 30000}}, + {"i32", {-2000000000, 5, 70000}}, + {"i64", {-9000000000000000000LL, 1, 5000000000LL}}, + {"f64", {1.5, -0.1, 2.5}}, + {"pairs", {{1.25, -3.75}, {1.25, -3.75}, json::array()}}, + {"mixed", {1, "a", 2.5, nullptr}}, + {"u16", {40000, 1, 65535}}, + {"u32", {4000000000U, 1, 70000}}, + {"u64", {18446744073709551615ULL, 1, 5000000000ULL}}, + {"nd", {{"_ArrayType_", "double"}, {"_ArraySize_", {2, 2}}, {"_ArrayData_", {1.5, -2.5, 3.5, -4.5}}}}, + {"nd8", {{"_ArrayType_", "uint8"}, {"_ArraySize_", {3}}, {"_ArrayData_", {1, 2, 3}}}}, + }; + const std::vector encoded = json::to_bjdata(document, true, true); + CHECK(json::from_bjdata(encoded) == document); + + for (std::size_t length = 0; length <= encoded.size(); ++length) + { + CAPTURE(length) + const std::vector contiguous(encoded.begin(), encoded.begin() + static_cast(length)); + const std::list bytewise(contiguous.begin(), contiguous.end()); + + CHECK(json::from_bjdata(contiguous, true, false).dump() == json::from_bjdata(bytewise.begin(), bytewise.end(), true, false).dump()); +#if !defined(JSON_NOEXCEPTION) + std::string contiguous_error; + std::string bytewise_error; + try + { + json _; + _ = json::from_bjdata(contiguous); + } + catch (const json::exception& e) + { + contiguous_error = e.what(); + } + try + { + json _; + _ = json::from_bjdata(bytewise.begin(), bytewise.end()); + } + catch (const json::exception& e) + { + bytewise_error = e.what(); + } + CHECK(contiguous_error == bytewise_error); +#endif + + SaxEventLogger from_contiguous; + SaxEventLogger from_bytewise; + CHECK(json::sax_parse(contiguous, &from_contiguous, json::input_format_t::bjdata) + == json::sax_parse(bytewise.begin(), bytewise.end(), &from_bytewise, json::input_format_t::bjdata)); + CHECK(from_contiguous.events == from_bytewise.events); + } + + for (int events = 0; events <= 80; ++events) + { + CAPTURE(events) + const std::list bytewise(encoded.begin(), encoded.end()); + SaxCountdown from_contiguous(events); + SaxCountdown from_bytewise(events); + CHECK(json::sax_parse(encoded, &from_contiguous, json::input_format_t::bjdata) + == json::sax_parse(bytewise.begin(), bytewise.end(), &from_bytewise, json::input_format_t::bjdata)); + } +} diff --git a/tests/src/unit-ubjson.cpp b/tests/src/unit-ubjson.cpp index 5c6378216..04983da1c 100644 --- a/tests/src/unit-ubjson.cpp +++ b/tests/src/unit-ubjson.cpp @@ -21,12 +21,15 @@ using nlohmann::json; #include #include +#include #include #include "make_test_data_available.hpp" #include "round_trip_corpus.hpp" #include "test_utils.hpp" #include "sax_countdown.hpp" +#include "sax_event_loggers.hpp" using utils::SaxCountdown; +using utils::SaxEventLogger; TEST_CASE("UBJSON") @@ -3400,3 +3403,72 @@ TEST_CASE("UBJSON optimized container type marker is validated for count 0") } } } + +TEST_CASE("UBJSON optimized arrays from contiguous and non-contiguous input") +{ + // contiguous input reads the numbers of an array with a type marker in + // one loop, other input number by number; both must give the same events, + // value, or error for the complete document and for every truncation of + // it, and stop at every event a SAX handler rejects + json document = + { + {"u8", {0, 200, 255}}, + {"i8", {-5, 100, -128}}, + {"i16", {-30000, 1000, 30000}}, + {"i32", {-2000000000, 5, 70000}}, + {"i64", {-9000000000000000000LL, 1, 5000000000LL}}, + {"f64", {1.5, -0.1, 2.5}}, + {"pairs", {{1.25, -3.75}, {1.25, -3.75}, json::array()}}, + {"mixed", {1, "a", 2.5, nullptr}}, + }; + const std::vector encoded = json::to_ubjson(document, true, true); + CHECK(json::from_ubjson(encoded) == document); + + for (std::size_t length = 0; length <= encoded.size(); ++length) + { + CAPTURE(length) + const std::vector contiguous(encoded.begin(), encoded.begin() + static_cast(length)); + const std::list bytewise(contiguous.begin(), contiguous.end()); + + CHECK(json::from_ubjson(contiguous, true, false).dump() == json::from_ubjson(bytewise.begin(), bytewise.end(), true, false).dump()); +#if !defined(JSON_NOEXCEPTION) + std::string contiguous_error; + std::string bytewise_error; + try + { + json _; + _ = json::from_ubjson(contiguous); + } + catch (const json::exception& e) + { + contiguous_error = e.what(); + } + try + { + json _; + _ = json::from_ubjson(bytewise.begin(), bytewise.end()); + } + catch (const json::exception& e) + { + bytewise_error = e.what(); + } + CHECK(contiguous_error == bytewise_error); +#endif + + SaxEventLogger from_contiguous; + SaxEventLogger from_bytewise; + CHECK(json::sax_parse(contiguous, &from_contiguous, json::input_format_t::ubjson) + == json::sax_parse(bytewise.begin(), bytewise.end(), &from_bytewise, json::input_format_t::ubjson)); + CHECK(from_contiguous.events == from_bytewise.events); + } + + for (int events = 0; events <= 80; ++events) + { + CAPTURE(events) + const std::list bytewise(encoded.begin(), encoded.end()); + SaxCountdown from_contiguous(events); + SaxCountdown from_bytewise(events); + CHECK(json::sax_parse(encoded, &from_contiguous, json::input_format_t::ubjson) + == json::sax_parse(bytewise.begin(), bytewise.end(), &from_bytewise, json::input_format_t::ubjson)); + } +} From 517444b504c4035e603a6500675dde4d28eaca59 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 1 Nov 2026 17:22:58 +0100 Subject: [PATCH 15/54] accept() checks strings without copying them (#5978) Signed-off-by: Niels Lohmann --- include/nlohmann/detail/input/lexer.hpp | 134 +++++++++++++------- include/nlohmann/detail/input/parser.hpp | 9 +- include/nlohmann/json.hpp | 11 +- single_include/nlohmann/json.hpp | 154 +++++++++++++++-------- tests/src/unit-class_parser.cpp | 61 +++++++++ 5 files changed, 265 insertions(+), 104 deletions(-) diff --git a/include/nlohmann/detail/input/lexer.hpp b/include/nlohmann/detail/input/lexer.hpp index 14d14f42a..66cb14b4f 100644 --- a/include/nlohmann/detail/input/lexer.hpp +++ b/include/nlohmann/detail/input/lexer.hpp @@ -209,8 +209,13 @@ constexpr bool input_adapter_bulk_window(std::false_type /*detected*/) @brief lexical analysis This class organizes the lexical analysis during JSON deserialization. + +DiscardValues: the caller only needs the token types and never looks at the values of number and string tokens (only +accept(), which parses through json_sax_acceptor). A template parameter rather than a member, so that the lexer of +parse() is compiled without the paths of accept(): with both string scans in one scan(), parse() was up to 2 % slower +on Apple M1. */ -template +template class lexer : public lexer_base { using number_integer_t = typename BasicJsonType::number_integer_t; @@ -268,10 +273,9 @@ class lexer : public lexer_base public: using token_type = typename lexer_base::token_type; - explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false, bool discard_number_values_ = false) noexcept + explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false) noexcept : ia(std::move(adapter)) , ignore_comments(ignore_comments_) - , discard_number_values(discard_number_values_) { // the token string of an error before the first token is everything // read so far @@ -388,6 +392,8 @@ class lexer : public lexer_base checks if it is inside the range. If a violation was detected, set up an error message and return false. Otherwise, return true. + @param[in] keep whether the bytes are added to token_buffer (false for + accept(), see scan()) @param[in] ranges list of integers; interpreted as list of pairs of inclusive lower and upper bound, respectively @@ -396,17 +402,18 @@ class lexer : public lexer_base @return true if and only if no range violation was detected */ - bool next_byte_in_range(std::initializer_list ranges) + template + bool next_byte_in_range(std::integral_constant keep, std::initializer_list ranges) { JSON_ASSERT(ranges.size() == 2 || ranges.size() == 4 || ranges.size() == 6); - add(current); + add_string_char(keep, current); for (auto range = ranges.begin(); range != ranges.end(); ++range) { get(); if (JSON_HEDLEY_LIKELY(*range <= current && current <= *(++range))) // NOLINT(bugprone-inc-dec-in-conditions) { - add(current); + add_string_char(keep, current); } else { @@ -419,14 +426,16 @@ class lexer : public lexer_base } /// contiguous input: bulk-append the run of ordinary characters and complete - /// well-formed UTF-8 sequences starting at the current read position, then + /// well-formed UTF-8 sequences starting at the current read position (only + /// skip it if @a keep is false), then /// read the first byte that needs individual handling (the closing quote, an /// escape, a control character, or an ill-formed UTF-8 byte) like get(). /// That byte is taken from the buffer that was just scanned. Through get(), /// Apple Clang merged this read with the other get() calls of the scan /// function, which load the read position again right after it was stored /// (3% of json::accept() on jeopardy.json on Apple M1). - char_int_type get_after_string_run(std::true_type /*bulk*/) + template + char_int_type get_after_string_run(std::true_type /*bulk*/, std::integral_constant keep) { // a pending unget must be consumed through the normal path first if (next_unget) @@ -444,7 +453,7 @@ class lexer : public lexer_base const std::size_t pos = lexer_string_bulk_run(data, remaining); if (pos != 0) { - token_buffer.append(reinterpret_cast(data), pos); + append_string_run(keep, data, pos); bulk_consume(reinterpret_cast(data), pos); } if (pos == remaining) @@ -488,7 +497,8 @@ class lexer : public lexer_base } /// streaming input: no bulk fast path - char_int_type get_after_string_run(std::false_type /*bulk*/) + template + char_int_type get_after_string_run(std::false_type /*bulk*/, std::integral_constant /*keep*/) { return get(); } @@ -502,13 +512,18 @@ class lexer : public lexer_base may contain \0 bytes), and token_buffer.size() is the number of bytes in the string. + @param[in] keep whether the string is copied into token_buffer; if false + (accept(), see scan()), it is only checked, with the same + errors, and token_buffer stays empty + @return token_type::value_string if string could be successfully scanned, token_type::parse_error otherwise @note In case of errors, variable error_message contains a textual description. */ - token_type scan_string() + template + token_type scan_string(std::integral_constant keep) { // reset token_buffer (ignore opening quote) reset(); @@ -520,7 +535,7 @@ class lexer : public lexer_base { // bulk-consume ordinary characters from contiguous input, then // get the next character, the one that ends the run - switch (get_after_string_run(std::integral_constant())) + switch (get_after_string_run(std::integral_constant(), keep)) { // end of file while parsing the string case char_traits::eof(): @@ -542,35 +557,35 @@ class lexer : public lexer_base { // quotation mark case '\"': - add('\"'); + add_string_char(keep, '\"'); break; // reverse solidus case '\\': - add('\\'); + add_string_char(keep, '\\'); break; // solidus case '/': - add('/'); + add_string_char(keep, '/'); break; // backspace case 'b': - add('\b'); + add_string_char(keep, '\b'); break; // form feed case 'f': - add('\f'); + add_string_char(keep, '\f'); break; // line feed case 'n': - add('\n'); + add_string_char(keep, '\n'); break; // carriage return case 'r': - add('\r'); + add_string_char(keep, '\r'); break; // tab case 't': - add('\t'); + add_string_char(keep, '\t'); break; // unicode escapes @@ -638,10 +653,7 @@ class lexer : public lexer_base JSON_ASSERT(0x00 <= codepoint && codepoint <= 0x10FFFF); // translate codepoint into bytes - encode_utf8(static_cast(codepoint), [this](std::uint32_t byte) - { - add(static_cast(byte)); - }); + add_codepoint(keep, codepoint); break; } @@ -824,7 +836,7 @@ class lexer : public lexer_base case 0x7E: case 0x7F: { - add(current); + add_string_char(keep, current); break; } @@ -860,7 +872,7 @@ class lexer : public lexer_base case 0xDE: case 0xDF: { - if (JSON_HEDLEY_UNLIKELY(!next_byte_in_range({0x80, 0xBF}))) + if (JSON_HEDLEY_UNLIKELY(!next_byte_in_range(keep, {0x80, 0xBF}))) { return token_type::parse_error; } @@ -870,7 +882,7 @@ class lexer : public lexer_base // U+0800..U+0FFF: bytes E0 A0..BF 80..BF case 0xE0: { - if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0xA0, 0xBF, 0x80, 0xBF})))) + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0xA0, 0xBF, 0x80, 0xBF})))) { return token_type::parse_error; } @@ -894,7 +906,7 @@ class lexer : public lexer_base case 0xEE: case 0xEF: { - if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0xBF, 0x80, 0xBF})))) + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0xBF, 0x80, 0xBF})))) { return token_type::parse_error; } @@ -904,7 +916,7 @@ class lexer : public lexer_base // U+D000..U+D7FF: bytes ED 80..9F 80..BF case 0xED: { - if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0x9F, 0x80, 0xBF})))) + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0x9F, 0x80, 0xBF})))) { return token_type::parse_error; } @@ -914,7 +926,7 @@ class lexer : public lexer_base // U+10000..U+3FFFF F0 90..BF 80..BF 80..BF case 0xF0: { - if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF})))) + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF})))) { return token_type::parse_error; } @@ -926,7 +938,7 @@ class lexer : public lexer_base case 0xF2: case 0xF3: { - if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF})))) + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF})))) { return token_type::parse_error; } @@ -936,7 +948,7 @@ class lexer : public lexer_base // U+100000..U+10FFFF F4 80..8F 80..BF 80..BF case 0xF4: { - if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF})))) + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF})))) { return token_type::parse_error; } @@ -1427,7 +1439,7 @@ scan_number_done: token_type convert_number(token_type number_type, std::size_t mantissa_end) { // accept() only needs to know whether the input is valid, so it sets - // discard_number_values (see json.hpp), and an integer token whose + // discard_values (see json.hpp), and an integer token whose // digit count shows that it fits is reported without calling // convert_integer(). A number with up to 18 digits always fits into // both std::uint64_t and std::int64_t (18 nines is about 1e18, below @@ -1442,7 +1454,7 @@ scan_number_done: // json_sax_acceptor, whose number callbacks discard their argument and // return true, and the parser rejects neither integers nor finite // floats. value_unsigned/value_integer are left unset here, so a caller - // that reads the converted value must not set discard_number_values. + // that reads the converted value must not set discard_values. // // On contiguous input, scan_number_bulk_contiguous() converts integer // tokens itself and does not pass them to this function, unless @@ -1450,7 +1462,7 @@ scan_number_done: // reached for input without bulk access (e.g. streams), with // JSON_DIAGNOSTIC_POSITIONS, or when scan_number_bulk_contiguous() // falls back to scan_number(). - if (discard_number_values) + if (discard_values()) { constexpr std::size_t safe_digit_count = 18; if (number_type == token_type::value_unsigned && token_buffer.size() <= safe_digit_count) @@ -1701,7 +1713,7 @@ scan_number_done: // w < 10^19, so a value below 10^max_exponent10 cannot overflow, and a // value that underflows is a finite number as well. Only a token that // may overflow has to be converted to find out. - if (JSON_HEDLEY_UNLIKELY(discard_number_values && s.exponent + 19 <= std::numeric_limits::max_exponent10)) + if (JSON_HEDLEY_UNLIKELY(discard_values() && s.exponent + 19 <= std::numeric_limits::max_exponent10)) { value_float = 0; return true; @@ -2033,6 +2045,34 @@ scan_number_done: token_buffer.push_back(static_cast(c)); } + /// add a character of a string token; accept() checks the string but does + /// not keep its value (@a keep is false) + void add_string_char(std::true_type /*keep*/, char_int_type c) + { + token_buffer.push_back(static_cast(c)); + } + + static void add_string_char(std::false_type /*keep*/, char_int_type /*c*/) noexcept {} + + /// add the UTF-8 bytes of a code point from a \\u escape to a string token + void add_codepoint(std::true_type /*keep*/, int codepoint) + { + encode_utf8(static_cast(codepoint), [this](std::uint32_t byte) + { + add(static_cast(byte)); + }); + } + + static void add_codepoint(std::false_type /*keep*/, int /*codepoint*/) noexcept {} + + /// append a run of ordinary string characters (see get_after_string_run()) + void append_string_run(std::true_type /*keep*/, const unsigned char* data, std::size_t n) + { + token_buffer.append(reinterpret_cast(data), n); + } + + static void append_string_run(std::false_type /*keep*/, const unsigned char* /*data*/, std::size_t /*n*/) noexcept {} + public: ///////////////////// // value getters @@ -2427,7 +2467,9 @@ scan_number_done: // string case '\"': - return scan_string(); + // accept() (DiscardValues) checks strings without copying them + // into token_buffer + return scan_string(keep_strings {}); // number case '-': @@ -2512,11 +2554,19 @@ scan_number_done: std::size_t decimal_point_position = std::string::npos; /// whether the caller only needs the token types and never looks at the - /// converted numeric values; set only by accept(), which parses through - /// json_sax_acceptor. When set, convert_number() skips converting integer - /// tokens whose digit count guarantees that they fit into 64 bits (see - /// there) - const bool discard_number_values = false; + /// values of number and string tokens; DiscardValues, set only by accept(), which parses + /// through json_sax_acceptor. When set, convert_number() skips converting + /// integer tokens whose digit count guarantees that they fit into 64 bits + /// (see there), floats are converted only when they may overflow (see + /// convert_accumulated_float()), and strings are checked without being + /// copied into token_buffer (see scan()) + static bool discard_values() noexcept + { + return DiscardValues; + } + + /// the tag of scan_string(): whether string tokens are kept + using keep_strings = typename std::conditional::type; }; } // namespace detail diff --git a/include/nlohmann/detail/input/parser.hpp b/include/nlohmann/detail/input/parser.hpp index 5bf8f1b6b..023327166 100644 --- a/include/nlohmann/detail/input/parser.hpp +++ b/include/nlohmann/detail/input/parser.hpp @@ -72,14 +72,14 @@ This class implements a parser for JSON text. Nested arrays and objects are trac stack instead of recursion, so deeply nested input does not exhaust the call stack, and what is read is reported as SAX events. */ -template +template class parser { using number_integer_t = typename BasicJsonType::number_integer_t; using number_unsigned_t = typename BasicJsonType::number_unsigned_t; using number_float_t = typename BasicJsonType::number_float_t; using string_t = typename BasicJsonType::string_t; - using lexer_t = lexer; + using lexer_t = lexer; using token_type = typename lexer_t::token_type; public: @@ -88,10 +88,9 @@ class parser parser_callback_t cb = nullptr, const bool allow_exceptions_ = true, const bool ignore_comments = false, - const bool ignore_trailing_commas_ = false, - const bool discard_number_values_ = false) + const bool ignore_trailing_commas_ = false) : callback(std::move(cb)) - , m_lexer(std::move(adapter), ignore_comments, discard_number_values_) + , m_lexer(std::move(adapter), ignore_comments) , allow_exceptions(allow_exceptions_) , ignore_trailing_commas(ignore_trailing_commas_) { diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 4d4f0a1cd..7bed813df 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -150,7 +150,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // can be restored when json_pointer backwards compatibility is removed // friend ::nlohmann::json_pointer; - template + template friend class ::nlohmann::detail::parser; friend ::nlohmann::detail::serializer; template @@ -183,12 +183,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec detail::parser_callback_tcb = nullptr, const bool allow_exceptions = true, const bool ignore_comments = false, - const bool ignore_trailing_commas = false, - const bool discard_number_values = false + const bool ignore_trailing_commas = false ) { return ::nlohmann::detail::parser(std::move(adapter), - std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas, discard_number_values); + std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas); } private: @@ -3117,7 +3116,9 @@ public: const bool ignore_comments, const bool ignore_trailing_commas) { - return parser(std::forward(adapter), nullptr, false, ignore_comments, ignore_trailing_commas, true).accept(true); + // the parser and lexer that discard the values of tokens (see lexer) + return ::nlohmann::detail::parser::type, true>(std::forward(adapter), + nullptr, false, ignore_comments, ignore_trailing_commas).accept(true); } /// generate SAX events from the input behind an already created adapter diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 84720a816..4bca3a3a5 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -13095,8 +13095,13 @@ constexpr bool input_adapter_bulk_window(std::false_type /*detected*/) @brief lexical analysis This class organizes the lexical analysis during JSON deserialization. + +DiscardValues: the caller only needs the token types and never looks at the values of number and string tokens (only +accept(), which parses through json_sax_acceptor). A template parameter rather than a member, so that the lexer of +parse() is compiled without the paths of accept(): with both string scans in one scan(), parse() was up to 2 % slower +on Apple M1. */ -template +template class lexer : public lexer_base { using number_integer_t = typename BasicJsonType::number_integer_t; @@ -13154,10 +13159,9 @@ class lexer : public lexer_base public: using token_type = typename lexer_base::token_type; - explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false, bool discard_number_values_ = false) noexcept + explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false) noexcept : ia(std::move(adapter)) , ignore_comments(ignore_comments_) - , discard_number_values(discard_number_values_) { // the token string of an error before the first token is everything // read so far @@ -13274,6 +13278,8 @@ class lexer : public lexer_base checks if it is inside the range. If a violation was detected, set up an error message and return false. Otherwise, return true. + @param[in] keep whether the bytes are added to token_buffer (false for + accept(), see scan()) @param[in] ranges list of integers; interpreted as list of pairs of inclusive lower and upper bound, respectively @@ -13282,17 +13288,18 @@ class lexer : public lexer_base @return true if and only if no range violation was detected */ - bool next_byte_in_range(std::initializer_list ranges) + template + bool next_byte_in_range(std::integral_constant keep, std::initializer_list ranges) { JSON_ASSERT(ranges.size() == 2 || ranges.size() == 4 || ranges.size() == 6); - add(current); + add_string_char(keep, current); for (auto range = ranges.begin(); range != ranges.end(); ++range) { get(); if (JSON_HEDLEY_LIKELY(*range <= current && current <= *(++range))) // NOLINT(bugprone-inc-dec-in-conditions) { - add(current); + add_string_char(keep, current); } else { @@ -13305,14 +13312,16 @@ class lexer : public lexer_base } /// contiguous input: bulk-append the run of ordinary characters and complete - /// well-formed UTF-8 sequences starting at the current read position, then + /// well-formed UTF-8 sequences starting at the current read position (only + /// skip it if @a keep is false), then /// read the first byte that needs individual handling (the closing quote, an /// escape, a control character, or an ill-formed UTF-8 byte) like get(). /// That byte is taken from the buffer that was just scanned. Through get(), /// Apple Clang merged this read with the other get() calls of the scan /// function, which load the read position again right after it was stored /// (3% of json::accept() on jeopardy.json on Apple M1). - char_int_type get_after_string_run(std::true_type /*bulk*/) + template + char_int_type get_after_string_run(std::true_type /*bulk*/, std::integral_constant keep) { // a pending unget must be consumed through the normal path first if (next_unget) @@ -13330,7 +13339,7 @@ class lexer : public lexer_base const std::size_t pos = lexer_string_bulk_run(data, remaining); if (pos != 0) { - token_buffer.append(reinterpret_cast(data), pos); + append_string_run(keep, data, pos); bulk_consume(reinterpret_cast(data), pos); } if (pos == remaining) @@ -13374,7 +13383,8 @@ class lexer : public lexer_base } /// streaming input: no bulk fast path - char_int_type get_after_string_run(std::false_type /*bulk*/) + template + char_int_type get_after_string_run(std::false_type /*bulk*/, std::integral_constant /*keep*/) { return get(); } @@ -13388,13 +13398,18 @@ class lexer : public lexer_base may contain \0 bytes), and token_buffer.size() is the number of bytes in the string. + @param[in] keep whether the string is copied into token_buffer; if false + (accept(), see scan()), it is only checked, with the same + errors, and token_buffer stays empty + @return token_type::value_string if string could be successfully scanned, token_type::parse_error otherwise @note In case of errors, variable error_message contains a textual description. */ - token_type scan_string() + template + token_type scan_string(std::integral_constant keep) { // reset token_buffer (ignore opening quote) reset(); @@ -13406,7 +13421,7 @@ class lexer : public lexer_base { // bulk-consume ordinary characters from contiguous input, then // get the next character, the one that ends the run - switch (get_after_string_run(std::integral_constant())) + switch (get_after_string_run(std::integral_constant(), keep)) { // end of file while parsing the string case char_traits::eof(): @@ -13428,35 +13443,35 @@ class lexer : public lexer_base { // quotation mark case '\"': - add('\"'); + add_string_char(keep, '\"'); break; // reverse solidus case '\\': - add('\\'); + add_string_char(keep, '\\'); break; // solidus case '/': - add('/'); + add_string_char(keep, '/'); break; // backspace case 'b': - add('\b'); + add_string_char(keep, '\b'); break; // form feed case 'f': - add('\f'); + add_string_char(keep, '\f'); break; // line feed case 'n': - add('\n'); + add_string_char(keep, '\n'); break; // carriage return case 'r': - add('\r'); + add_string_char(keep, '\r'); break; // tab case 't': - add('\t'); + add_string_char(keep, '\t'); break; // unicode escapes @@ -13524,10 +13539,7 @@ class lexer : public lexer_base JSON_ASSERT(0x00 <= codepoint && codepoint <= 0x10FFFF); // translate codepoint into bytes - encode_utf8(static_cast(codepoint), [this](std::uint32_t byte) - { - add(static_cast(byte)); - }); + add_codepoint(keep, codepoint); break; } @@ -13710,7 +13722,7 @@ class lexer : public lexer_base case 0x7E: case 0x7F: { - add(current); + add_string_char(keep, current); break; } @@ -13746,7 +13758,7 @@ class lexer : public lexer_base case 0xDE: case 0xDF: { - if (JSON_HEDLEY_UNLIKELY(!next_byte_in_range({0x80, 0xBF}))) + if (JSON_HEDLEY_UNLIKELY(!next_byte_in_range(keep, {0x80, 0xBF}))) { return token_type::parse_error; } @@ -13756,7 +13768,7 @@ class lexer : public lexer_base // U+0800..U+0FFF: bytes E0 A0..BF 80..BF case 0xE0: { - if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0xA0, 0xBF, 0x80, 0xBF})))) + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0xA0, 0xBF, 0x80, 0xBF})))) { return token_type::parse_error; } @@ -13780,7 +13792,7 @@ class lexer : public lexer_base case 0xEE: case 0xEF: { - if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0xBF, 0x80, 0xBF})))) + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0xBF, 0x80, 0xBF})))) { return token_type::parse_error; } @@ -13790,7 +13802,7 @@ class lexer : public lexer_base // U+D000..U+D7FF: bytes ED 80..9F 80..BF case 0xED: { - if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0x9F, 0x80, 0xBF})))) + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0x9F, 0x80, 0xBF})))) { return token_type::parse_error; } @@ -13800,7 +13812,7 @@ class lexer : public lexer_base // U+10000..U+3FFFF F0 90..BF 80..BF 80..BF case 0xF0: { - if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF})))) + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF})))) { return token_type::parse_error; } @@ -13812,7 +13824,7 @@ class lexer : public lexer_base case 0xF2: case 0xF3: { - if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF})))) + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF})))) { return token_type::parse_error; } @@ -13822,7 +13834,7 @@ class lexer : public lexer_base // U+100000..U+10FFFF F4 80..8F 80..BF 80..BF case 0xF4: { - if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF})))) + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF})))) { return token_type::parse_error; } @@ -14313,7 +14325,7 @@ scan_number_done: token_type convert_number(token_type number_type, std::size_t mantissa_end) { // accept() only needs to know whether the input is valid, so it sets - // discard_number_values (see json.hpp), and an integer token whose + // discard_values (see json.hpp), and an integer token whose // digit count shows that it fits is reported without calling // convert_integer(). A number with up to 18 digits always fits into // both std::uint64_t and std::int64_t (18 nines is about 1e18, below @@ -14328,7 +14340,7 @@ scan_number_done: // json_sax_acceptor, whose number callbacks discard their argument and // return true, and the parser rejects neither integers nor finite // floats. value_unsigned/value_integer are left unset here, so a caller - // that reads the converted value must not set discard_number_values. + // that reads the converted value must not set discard_values. // // On contiguous input, scan_number_bulk_contiguous() converts integer // tokens itself and does not pass them to this function, unless @@ -14336,7 +14348,7 @@ scan_number_done: // reached for input without bulk access (e.g. streams), with // JSON_DIAGNOSTIC_POSITIONS, or when scan_number_bulk_contiguous() // falls back to scan_number(). - if (discard_number_values) + if (discard_values()) { constexpr std::size_t safe_digit_count = 18; if (number_type == token_type::value_unsigned && token_buffer.size() <= safe_digit_count) @@ -14587,7 +14599,7 @@ scan_number_done: // w < 10^19, so a value below 10^max_exponent10 cannot overflow, and a // value that underflows is a finite number as well. Only a token that // may overflow has to be converted to find out. - if (JSON_HEDLEY_UNLIKELY(discard_number_values && s.exponent + 19 <= std::numeric_limits::max_exponent10)) + if (JSON_HEDLEY_UNLIKELY(discard_values() && s.exponent + 19 <= std::numeric_limits::max_exponent10)) { value_float = 0; return true; @@ -14919,6 +14931,34 @@ scan_number_done: token_buffer.push_back(static_cast(c)); } + /// add a character of a string token; accept() checks the string but does + /// not keep its value (@a keep is false) + void add_string_char(std::true_type /*keep*/, char_int_type c) + { + token_buffer.push_back(static_cast(c)); + } + + static void add_string_char(std::false_type /*keep*/, char_int_type /*c*/) noexcept {} + + /// add the UTF-8 bytes of a code point from a \\u escape to a string token + void add_codepoint(std::true_type /*keep*/, int codepoint) + { + encode_utf8(static_cast(codepoint), [this](std::uint32_t byte) + { + add(static_cast(byte)); + }); + } + + static void add_codepoint(std::false_type /*keep*/, int /*codepoint*/) noexcept {} + + /// append a run of ordinary string characters (see get_after_string_run()) + void append_string_run(std::true_type /*keep*/, const unsigned char* data, std::size_t n) + { + token_buffer.append(reinterpret_cast(data), n); + } + + static void append_string_run(std::false_type /*keep*/, const unsigned char* /*data*/, std::size_t /*n*/) noexcept {} + public: ///////////////////// // value getters @@ -15313,7 +15353,9 @@ scan_number_done: // string case '\"': - return scan_string(); + // accept() (DiscardValues) checks strings without copying them + // into token_buffer + return scan_string(keep_strings {}); // number case '-': @@ -15398,11 +15440,19 @@ scan_number_done: std::size_t decimal_point_position = std::string::npos; /// whether the caller only needs the token types and never looks at the - /// converted numeric values; set only by accept(), which parses through - /// json_sax_acceptor. When set, convert_number() skips converting integer - /// tokens whose digit count guarantees that they fit into 64 bits (see - /// there) - const bool discard_number_values = false; + /// values of number and string tokens; DiscardValues, set only by accept(), which parses + /// through json_sax_acceptor. When set, convert_number() skips converting + /// integer tokens whose digit count guarantees that they fit into 64 bits + /// (see there), floats are converted only when they may overflow (see + /// convert_accumulated_float()), and strings are checked without being + /// copied into token_buffer (see scan()) + static bool discard_values() noexcept + { + return DiscardValues; + } + + /// the tag of scan_string(): whether string tokens are kept + using keep_strings = typename std::conditional::type; }; } // namespace detail @@ -23023,14 +23073,14 @@ This class implements a parser for JSON text. Nested arrays and objects are trac stack instead of recursion, so deeply nested input does not exhaust the call stack, and what is read is reported as SAX events. */ -template +template class parser { using number_integer_t = typename BasicJsonType::number_integer_t; using number_unsigned_t = typename BasicJsonType::number_unsigned_t; using number_float_t = typename BasicJsonType::number_float_t; using string_t = typename BasicJsonType::string_t; - using lexer_t = lexer; + using lexer_t = lexer; using token_type = typename lexer_t::token_type; public: @@ -23039,10 +23089,9 @@ class parser parser_callback_t cb = nullptr, const bool allow_exceptions_ = true, const bool ignore_comments = false, - const bool ignore_trailing_commas_ = false, - const bool discard_number_values_ = false) + const bool ignore_trailing_commas_ = false) : callback(std::move(cb)) - , m_lexer(std::move(adapter), ignore_comments, discard_number_values_) + , m_lexer(std::move(adapter), ignore_comments) , allow_exceptions(allow_exceptions_) , ignore_trailing_commas(ignore_trailing_commas_) { @@ -37439,7 +37488,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // can be restored when json_pointer backwards compatibility is removed // friend ::nlohmann::json_pointer; - template + template friend class ::nlohmann::detail::parser; friend ::nlohmann::detail::serializer; template @@ -37472,12 +37521,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec detail::parser_callback_tcb = nullptr, const bool allow_exceptions = true, const bool ignore_comments = false, - const bool ignore_trailing_commas = false, - const bool discard_number_values = false + const bool ignore_trailing_commas = false ) { return ::nlohmann::detail::parser(std::move(adapter), - std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas, discard_number_values); + std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas); } private: @@ -40406,7 +40454,9 @@ public: const bool ignore_comments, const bool ignore_trailing_commas) { - return parser(std::forward(adapter), nullptr, false, ignore_comments, ignore_trailing_commas, true).accept(true); + // the parser and lexer that discard the values of tokens (see lexer) + return ::nlohmann::detail::parser::type, true>(std::forward(adapter), + nullptr, false, ignore_comments, ignore_trailing_commas).accept(true); } /// generate SAX events from the input behind an already created adapter diff --git a/tests/src/unit-class_parser.cpp b/tests/src/unit-class_parser.cpp index 090ab4b41..987e25858 100644 --- a/tests/src/unit-class_parser.cpp +++ b/tests/src/unit-class_parser.cpp @@ -2982,6 +2982,67 @@ TEST_CASE("accept() agrees with parse() on floats around the overflow limit") } } +TEST_CASE("accept() agrees with parse() on strings, which it checks without copying them") +{ + // accept() scans strings and object keys with a separate instantiation of + // the string scanner that does not copy them into the token buffer; it + // must accept and reject exactly what parse() does + const std::vector pieces = + { + "", "abc", R"(\"\\\/\b\f\n\r\t)", R"(\u0041\u00e4\u20AC)", "\\ud83d\\ude00", "\\uD83D\\uDE00", + "\\ud800", "\\udc00", "\\ud800\\u0041", "\\ud800x", "\\udbff\\udfff", "\\u12", "\\u12g4", "\\x", "\\", + "\xc3\xa4", "\xe2\x82\xac", "\xf0\x9f\x98\x80", "\xf4\x8f\xbf\xbf", "\xc0\xaf", "\xed\xa0\x80", "\xf4\x90\x80\x80", + "\xff", "\xe2\x82", "\xc3", "\x01", "\x1f", "\x7f", "\n", std::string(1, '\0') + }; + + for (std::size_t run = 0; run <= 70; ++run) + { + for (const std::string& piece : pieces) + { + // a run of ordinary characters puts the piece at every offset of the bulk scanners + const std::string str = std::string(run, 'a') + piece; + std::string opened = "\""; + opened += str; + std::string closed = opened; + closed += '"'; + std::vector texts; + texts.push_back(closed); + texts.emplace_back("["); + texts.back() += closed; + texts.back() += ']'; + texts.emplace_back("{"); + texts.back() += closed; + texts.back() += ":1}"; + texts.emplace_back("{\"k\":"); + texts.back() += closed; + texts.back() += ','; + texts.back() += closed; + texts.back() += ":["; + texts.back() += closed; + texts.back() += "]}"; + texts.emplace_back("["); + texts.back() += opened; + for (const std::string& text : texts) + { + CAPTURE(text) + for (int options = 0; options < 4; ++options) + { + const bool ignore_comments = (options & 1) != 0; + const bool ignore_trailing_commas = (options & 2) != 0; + const bool parses = !json::parse(text, nullptr, false, ignore_comments, ignore_trailing_commas).is_discarded(); + CHECK(json::accept(text, ignore_comments, ignore_trailing_commas) == parses); + + std::istringstream stream(text); + CHECK(json::accept(stream, ignore_comments, ignore_trailing_commas) == parses); + + const std::list chars(text.begin(), text.end()); + CHECK(json::accept(chars.begin(), chars.end(), ignore_comments, ignore_trailing_commas) == parses); + } + } + } + } +} + namespace { // a DOM parser that reports parse errors without throwing, as in the sax_no_exception example From 9742edc294a0ae08fac79539da02ebef8f946948 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 1 Nov 2026 17:23:12 +0100 Subject: [PATCH 16/54] BSON writer: write to_bson(j) in one pass, with sizes written afterwards (#5979) Signed-off-by: Niels Lohmann --- .../nlohmann/detail/output/binary_writer.hpp | 229 +++++++++++++++- .../detail/output/output_adapters.hpp | 26 ++ single_include/nlohmann/json.hpp | 255 +++++++++++++++++- tests/src/unit-bson.cpp | 151 +++++++++++ 4 files changed, 659 insertions(+), 2 deletions(-) diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 8732e52b4..9d21d2a7c 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -2088,7 +2088,8 @@ class binary_writer /// arrays: the first index with one more digit; only maintained by /// @ref calc_bson_sizes std::uint64_t index_boundary = 10; - /// @ref calc_bson_sizes only: where its size goes in the table + /// @ref calc_bson_sizes: where its size goes in the table; @ref + /// write_bson_document_single_pass: the position of its size field std::size_t size_slot; /// @ref calc_bson_sizes only: the size of its entries seen so far std::size_t entries_size = 0; @@ -2339,6 +2340,199 @@ class binary_writer } } + template + using sink_written_t = decltype(std::declval().written()); + + /// whether every integer fits the 64 bits BSON has, so that writing an + /// array of numbers needs no range check + static constexpr bool bson_integers_fit_64_bits = + std::numeric_limits::is_specialized + && std::numeric_limits::digits <= std::numeric_limits::digits + && std::numeric_limits::is_specialized + && std::numeric_limits::digits <= std::numeric_limits::digits; + + /*! + @brief whether BSON documents can be written in a single pass: the sink + can overwrite what it has written (output_vector_sink), and + @ref bson_integers_fit_64_bits + */ + using bson_single_pass_sink = conjunction, std::integral_constant>; + + /// write a placeholder for the size of an object or array whose entries + /// follow; @return its position, for @ref close_bson_container + std::size_t write_bson_size_placeholder() + { + const std::size_t pos = oa.written(); + // zero in any byte order + const std::array zero{}; + oa.write_characters(zero.data(), zero.size()); + return pos; + } + + /// end the object or array whose size is at @a pos and write its size + /// there: everything from that field to the terminating null byte + /// @return false (and the size is not written) if it does not fit + bool close_bson_container(const std::size_t pos) + { + oa.write_character(to_char_type(0x00)); + const std::size_t size = oa.written() - pos; + if (JSON_HEDLEY_UNLIKELY(!value_in_range_of(size))) + { + return false; + } + const auto le_size = static_cast(size); + oa.overwrite(pos, is_little_endian ? le_size : byteswap(le_size)); + return true; + } + + /*! + @brief check the value @a j of a BSON entry as @ref calc_bson_value_size + does, for @ref write_bson_document_single_pass + @return at least the size of the value + A number, boolean, or null needs no check there (its integers have at most + 64 bits) and has at most 8 characters; a discarded value is rejected with + the same exception when it is written. + */ + std::size_t check_bson_value(const BasicJsonType& j) + { + return (j.is_string() || j.is_binary()) ? calc_bson_value_size(j) : 8; + } + + /// whether an entry of @a entry_size characters keeps the document that + /// begins at @a start, closed by @a open_containers null bytes, within + /// the 32-bit size of BSON + bool bson_entry_fits(const std::size_t start, const std::size_t entry_size, const std::size_t open_containers) const noexcept + { + const auto limit = static_cast((std::numeric_limits::max)()); + const std::size_t written = oa.written() - start; + return written <= limit && entry_size <= limit - written && open_containers <= limit - written - entry_size; + } + + /*! + @brief Serializes the JSON object @a document as a BSON document in a + single pass, writing the size of each object and array once it is + complete + + Checks every key and value as @ref calc_bson_sizes does, in the same + order, so the same exception is thrown; the output written up to then is + discarded with the vector it was written to. + + @return false as soon as the document or a string, binary value, object, + or array in it would be larger than the 32-bit BSON sizes allow; + nothing large has been written then, and the caller writes the + document in two passes, which reports that error + */ + template::value, int>::type = 0> + bool write_bson_document_single_pass(const BasicJsonType& document) + { + const std::size_t start = oa.written(); + // the object or array whose entries are being written (with the + // position of its size), and the ones it is in + bson_frame current(&document, write_bson_size_placeholder()); + std::vector parents; + + while (true) + { + const BasicJsonType* nested = nullptr; + std::size_t nested_size_pos = 0; + if (current.value->is_object()) + { + const auto& object = *current.value->m_data.m_value.object; + while (nested == nullptr && current.member != object.cend()) + { + const auto& el = *current.member; + ++current.member; + const std::size_t header_size = calc_bson_entry_header_size(el.first, el.second); + const std::size_t value_size = el.second.is_structured() ? 0 : check_bson_value(el.second); + if (JSON_HEDLEY_UNLIKELY(!bson_entry_fits(start, header_size + value_size, parents.size() + 1))) + { + return false; + } + if (el.second.is_structured()) + { + write_bson_entry_header(el.first, el.second.is_object() ? 0x03 : 0x04); + nested_size_pos = write_bson_size_placeholder(); + nested = &el.second; + } + else + { + write_bson_value(el.first, el.second); + } + } + } + else + { + const auto& array = *current.value->m_data.m_value.array; + while (nested == nullptr && current.index < array.size()) + { + if (is_bson_fixed_size(array[current.index])) + { + write_bson_array_fixed_entries(current); + if (current.index == array.size()) + { + break; + } + } + + // a string, binary value, object or array + const BasicJsonType& el = array[current.index]; + const string_t index_name(current.index_name.data(), current.index_name_size); + current.next_index(); + const std::size_t value_size = el.is_structured() ? 0 : check_bson_value(el); + if (JSON_HEDLEY_UNLIKELY(!bson_entry_fits(start, 2 + index_name.size() + value_size, parents.size() + 1))) + { + return false; + } + if (el.is_structured()) + { + write_bson_entry_header(index_name, el.is_object() ? 0x03 : 0x04); + nested_size_pos = write_bson_size_placeholder(); + nested = ⪙ + } + else + { + write_bson_value(index_name, el); + } + } + } + + if (nested != nullptr) + { + if (nested->is_array()) + { + // an array of booleans, numbers and null is written without a frame on the stack + bson_frame leaf(nested, nested_size_pos); + write_bson_array_fixed_entries(leaf); + if (leaf.index == nested->m_data.m_value.array->size()) + { + if (JSON_HEDLEY_UNLIKELY(!close_bson_container(nested_size_pos))) + { + return false; + } + continue; + } + parents.push_back(std::move(current)); + current = std::move(leaf); + continue; + } + parents.push_back(std::move(current)); + current = bson_frame(nested, nested_size_pos); + continue; + } + + if (JSON_HEDLEY_UNLIKELY(!close_bson_container(current.size_slot))) + { + return false; + } + if (parents.empty()) + { + return true; + } + current = std::move(parents.back()); + parents.pop_back(); + } + } + /*! @brief Serializes the JSON object @a document as a BSON document @@ -2350,6 +2544,39 @@ class binary_writer @pre document.type() == value_t::object */ void write_bson_document(const BasicJsonType& document) + { + write_bson_document(document, bson_single_pass_sink()); + } + + /// the sink can overwrite: one pass, unless the error handler replaces or + /// drops ill-formed UTF-8, which changes the sizes the checks compute + void write_bson_document(const BasicJsonType& document, std::true_type /*single_pass*/) + { + const std::size_t start = oa.written(); + if ((error_handler == error_handler_t::keep || error_handler == error_handler_t::strict) + && write_bson_document_single_pass(document)) + { + return; + } + // a length does not fit: start over, so that it is reported as the + // two passes report it, before anything is written + oa.rewind(start); + write_bson_document_two_pass(document); + } + + void write_bson_document(const BasicJsonType& document, std::false_type /*single_pass*/) + { + write_bson_document_two_pass(document); + } + + /*! + @brief Serializes the JSON object @a document as a BSON document: the + sizes of all objects and arrays first, then the entries + + @param[in] document the JSON object to serialize + @pre document.type() == value_t::object + */ + void write_bson_document_two_pass(const BasicJsonType& document) { std::vector nested_sizes; const std::size_t document_size = calc_bson_sizes(document, nested_sizes); diff --git a/include/nlohmann/detail/output/output_adapters.hpp b/include/nlohmann/detail/output/output_adapters.hpp index e40d98548..e9e57ab9a 100644 --- a/include/nlohmann/detail/output/output_adapters.hpp +++ b/include/nlohmann/detail/output/output_adapters.hpp @@ -11,6 +11,7 @@ #include // max #include // array #include // size_t +#include // uint32_t #include // memcpy #include // shared_ptr, make_shared #include // basic_string @@ -227,6 +228,31 @@ class output_vector_sink void flush() noexcept {} + /// the number of characters written so far, the position of the next one + std::size_t written() const noexcept + { + return position(); + } + + /// @brief replace the 4 bytes written before at position @a pos with + /// those of @a n, in the byte order of the machine + /// + /// Lets a writer put a length in front of data it has written since, such + /// as the size of a BSON document, without computing it in advance. + void overwrite(const std::size_t pos, const std::uint32_t n) noexcept + { + static_assert(sizeof(CharType) == 1, "overwrite() writes bytes"); + JSON_ASSERT(pos + sizeof(n) <= position()); + std::memcpy(v->data() + pos, &n, sizeof(n)); + } + + /// drop the characters written from position @a pos on + void rewind(const std::size_t pos) noexcept + { + JSON_ASSERT(pos <= position()); + cursor = v->data() + pos; + } + private: /// number of characters of *v written so far std::size_t position() const noexcept diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 4bca3a3a5..3c2670039 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -27161,6 +27161,7 @@ NLOHMANN_JSON_NAMESPACE_END #include // max #include // array #include // size_t +#include // uint32_t #include // memcpy #include // shared_ptr, make_shared #include // basic_string @@ -27378,6 +27379,31 @@ class output_vector_sink void flush() noexcept {} + /// the number of characters written so far, the position of the next one + std::size_t written() const noexcept + { + return position(); + } + + /// @brief replace the 4 bytes written before at position @a pos with + /// those of @a n, in the byte order of the machine + /// + /// Lets a writer put a length in front of data it has written since, such + /// as the size of a BSON document, without computing it in advance. + void overwrite(const std::size_t pos, const std::uint32_t n) noexcept + { + static_assert(sizeof(CharType) == 1, "overwrite() writes bytes"); + JSON_ASSERT(pos + sizeof(n) <= position()); + std::memcpy(v->data() + pos, &n, sizeof(n)); + } + + /// drop the characters written from position @a pos on + void rewind(const std::size_t pos) noexcept + { + JSON_ASSERT(pos <= position()); + cursor = v->data() + pos; + } + private: /// number of characters of *v written so far std::size_t position() const noexcept @@ -29822,7 +29848,8 @@ class binary_writer /// arrays: the first index with one more digit; only maintained by /// @ref calc_bson_sizes std::uint64_t index_boundary = 10; - /// @ref calc_bson_sizes only: where its size goes in the table + /// @ref calc_bson_sizes: where its size goes in the table; @ref + /// write_bson_document_single_pass: the position of its size field std::size_t size_slot; /// @ref calc_bson_sizes only: the size of its entries seen so far std::size_t entries_size = 0; @@ -30073,6 +30100,199 @@ class binary_writer } } + template + using sink_written_t = decltype(std::declval().written()); + + /// whether every integer fits the 64 bits BSON has, so that writing an + /// array of numbers needs no range check + static constexpr bool bson_integers_fit_64_bits = + std::numeric_limits::is_specialized + && std::numeric_limits::digits <= std::numeric_limits::digits + && std::numeric_limits::is_specialized + && std::numeric_limits::digits <= std::numeric_limits::digits; + + /*! + @brief whether BSON documents can be written in a single pass: the sink + can overwrite what it has written (output_vector_sink), and + @ref bson_integers_fit_64_bits + */ + using bson_single_pass_sink = conjunction, std::integral_constant>; + + /// write a placeholder for the size of an object or array whose entries + /// follow; @return its position, for @ref close_bson_container + std::size_t write_bson_size_placeholder() + { + const std::size_t pos = oa.written(); + // zero in any byte order + const std::array zero{}; + oa.write_characters(zero.data(), zero.size()); + return pos; + } + + /// end the object or array whose size is at @a pos and write its size + /// there: everything from that field to the terminating null byte + /// @return false (and the size is not written) if it does not fit + bool close_bson_container(const std::size_t pos) + { + oa.write_character(to_char_type(0x00)); + const std::size_t size = oa.written() - pos; + if (JSON_HEDLEY_UNLIKELY(!value_in_range_of(size))) + { + return false; + } + const auto le_size = static_cast(size); + oa.overwrite(pos, is_little_endian ? le_size : byteswap(le_size)); + return true; + } + + /*! + @brief check the value @a j of a BSON entry as @ref calc_bson_value_size + does, for @ref write_bson_document_single_pass + @return at least the size of the value + A number, boolean, or null needs no check there (its integers have at most + 64 bits) and has at most 8 characters; a discarded value is rejected with + the same exception when it is written. + */ + std::size_t check_bson_value(const BasicJsonType& j) + { + return (j.is_string() || j.is_binary()) ? calc_bson_value_size(j) : 8; + } + + /// whether an entry of @a entry_size characters keeps the document that + /// begins at @a start, closed by @a open_containers null bytes, within + /// the 32-bit size of BSON + bool bson_entry_fits(const std::size_t start, const std::size_t entry_size, const std::size_t open_containers) const noexcept + { + const auto limit = static_cast((std::numeric_limits::max)()); + const std::size_t written = oa.written() - start; + return written <= limit && entry_size <= limit - written && open_containers <= limit - written - entry_size; + } + + /*! + @brief Serializes the JSON object @a document as a BSON document in a + single pass, writing the size of each object and array once it is + complete + + Checks every key and value as @ref calc_bson_sizes does, in the same + order, so the same exception is thrown; the output written up to then is + discarded with the vector it was written to. + + @return false as soon as the document or a string, binary value, object, + or array in it would be larger than the 32-bit BSON sizes allow; + nothing large has been written then, and the caller writes the + document in two passes, which reports that error + */ + template::value, int>::type = 0> + bool write_bson_document_single_pass(const BasicJsonType& document) + { + const std::size_t start = oa.written(); + // the object or array whose entries are being written (with the + // position of its size), and the ones it is in + bson_frame current(&document, write_bson_size_placeholder()); + std::vector parents; + + while (true) + { + const BasicJsonType* nested = nullptr; + std::size_t nested_size_pos = 0; + if (current.value->is_object()) + { + const auto& object = *current.value->m_data.m_value.object; + while (nested == nullptr && current.member != object.cend()) + { + const auto& el = *current.member; + ++current.member; + const std::size_t header_size = calc_bson_entry_header_size(el.first, el.second); + const std::size_t value_size = el.second.is_structured() ? 0 : check_bson_value(el.second); + if (JSON_HEDLEY_UNLIKELY(!bson_entry_fits(start, header_size + value_size, parents.size() + 1))) + { + return false; + } + if (el.second.is_structured()) + { + write_bson_entry_header(el.first, el.second.is_object() ? 0x03 : 0x04); + nested_size_pos = write_bson_size_placeholder(); + nested = &el.second; + } + else + { + write_bson_value(el.first, el.second); + } + } + } + else + { + const auto& array = *current.value->m_data.m_value.array; + while (nested == nullptr && current.index < array.size()) + { + if (is_bson_fixed_size(array[current.index])) + { + write_bson_array_fixed_entries(current); + if (current.index == array.size()) + { + break; + } + } + + // a string, binary value, object or array + const BasicJsonType& el = array[current.index]; + const string_t index_name(current.index_name.data(), current.index_name_size); + current.next_index(); + const std::size_t value_size = el.is_structured() ? 0 : check_bson_value(el); + if (JSON_HEDLEY_UNLIKELY(!bson_entry_fits(start, 2 + index_name.size() + value_size, parents.size() + 1))) + { + return false; + } + if (el.is_structured()) + { + write_bson_entry_header(index_name, el.is_object() ? 0x03 : 0x04); + nested_size_pos = write_bson_size_placeholder(); + nested = ⪙ + } + else + { + write_bson_value(index_name, el); + } + } + } + + if (nested != nullptr) + { + if (nested->is_array()) + { + // an array of booleans, numbers and null is written without a frame on the stack + bson_frame leaf(nested, nested_size_pos); + write_bson_array_fixed_entries(leaf); + if (leaf.index == nested->m_data.m_value.array->size()) + { + if (JSON_HEDLEY_UNLIKELY(!close_bson_container(nested_size_pos))) + { + return false; + } + continue; + } + parents.push_back(std::move(current)); + current = std::move(leaf); + continue; + } + parents.push_back(std::move(current)); + current = bson_frame(nested, nested_size_pos); + continue; + } + + if (JSON_HEDLEY_UNLIKELY(!close_bson_container(current.size_slot))) + { + return false; + } + if (parents.empty()) + { + return true; + } + current = std::move(parents.back()); + parents.pop_back(); + } + } + /*! @brief Serializes the JSON object @a document as a BSON document @@ -30084,6 +30304,39 @@ class binary_writer @pre document.type() == value_t::object */ void write_bson_document(const BasicJsonType& document) + { + write_bson_document(document, bson_single_pass_sink()); + } + + /// the sink can overwrite: one pass, unless the error handler replaces or + /// drops ill-formed UTF-8, which changes the sizes the checks compute + void write_bson_document(const BasicJsonType& document, std::true_type /*single_pass*/) + { + const std::size_t start = oa.written(); + if ((error_handler == error_handler_t::keep || error_handler == error_handler_t::strict) + && write_bson_document_single_pass(document)) + { + return; + } + // a length does not fit: start over, so that it is reported as the + // two passes report it, before anything is written + oa.rewind(start); + write_bson_document_two_pass(document); + } + + void write_bson_document(const BasicJsonType& document, std::false_type /*single_pass*/) + { + write_bson_document_two_pass(document); + } + + /*! + @brief Serializes the JSON object @a document as a BSON document: the + sizes of all objects and arrays first, then the entries + + @param[in] document the JSON object to serialize + @pre document.type() == value_t::object + */ + void write_bson_document_two_pass(const BasicJsonType& document) { std::vector nested_sizes; const std::size_t document_size = calc_bson_sizes(document, nested_sizes); diff --git a/tests/src/unit-bson.cpp b/tests/src/unit-bson.cpp index f93e0a71b..f1f43ccd7 100644 --- a/tests/src/unit-bson.cpp +++ b/tests/src/unit-bson.cpp @@ -19,6 +19,7 @@ #include using nlohmann::json; +#include #include #include #include @@ -2300,3 +2301,153 @@ TEST_CASE("BSON large strings and binaries (chunked reader)") } } } + +TEST_CASE("BSON written in one pass and in two passes") +{ + // to_bson() into a returned vector writes each size once the object or + // array is complete; through an output adapter, all sizes are computed + // first. Both must produce the same bytes and throw the same exceptions. + const auto two_pass = [](const json & j, const json::error_handler_t handler) + { + std::vector result; + json::to_bson(j, result, handler); + return result; + }; + + const auto check_same = [&](const json & j, const json::error_handler_t handler) + { + CAPTURE(j.dump(-1, ' ', true, json::error_handler_t::replace)) + CAPTURE(static_cast(handler)) +#if !defined(JSON_NOEXCEPTION) + std::string one_pass_error; + std::string two_pass_error; + std::vector one; + std::vector two; + try + { + one = json::to_bson(j, handler); + } + catch (const json::exception& e) + { + one_pass_error = e.what(); + } + try + { + two = two_pass(j, handler); + } + catch (const json::exception& e) + { + two_pass_error = e.what(); + } + CHECK(one_pass_error == two_pass_error); + CHECK(one == two); +#else + CHECK(json::to_bson(j, handler) == two_pass(j, handler)); +#endif + }; + + const std::array handlers = {{ + json::error_handler_t::keep, json::error_handler_t::strict, + json::error_handler_t::replace, json::error_handler_t::ignore + } + }; + + SECTION("corpus") + { + for (const auto& value : utils::round_trip_corpus::values()) + { + const json document = value.is_object() ? value : json{{"value", value}}; + for (const auto handler : handlers) + { + check_same(document, handler); + } + } + } + + SECTION("nesting and arrays of every length class") + { + json document = json::object(); + document["empty object"] = json::object(); + document["empty array"] = json::array(); + for (const std::size_t n : + { + 1u, 9u, 10u, 11u, 99u, 100u, 101u, 1000u + }) + { + json numbers = json::array(); + json mixed = json::array(); + for (std::size_t i = 0; i < n; ++i) + { + numbers.push_back(i % 3 == 0 ? json(static_cast(i) / 3) : json(static_cast(i) * 1000003)); + switch (i % 4) + { + case 0: + mixed.push_back(json("s" + std::to_string(i))); + break; + case 1: + mixed.push_back(json::array({i, json::object({{"k", i}})})); + break; + case 2: + mixed.push_back(json(nullptr)); + break; + default: + mixed.push_back(json(i % 2 == 0)); + break; + } + } + document["numbers " + std::to_string(n)] = numbers; + document["mixed " + std::to_string(n)] = mixed; + document["nested " + std::to_string(n)] = json::object({{"a", numbers}, {"b", json::array({mixed, json::object()})}}); + } + document["binary"] = json::binary({1, 2, 3}, 0x80); + document["binary old"] = json::binary({1, 2, 3}, 0x02); + for (const auto handler : handlers) + { + check_same(document, handler); + } + } + +#if !defined(JSON_NOEXCEPTION) + // every entry here throws, which aborts without exceptions + SECTION("errors at every position") + { + // an entry that cannot be written, at each place among valid ones + const std::string bad_utf8 = "\xC3\x28"; + const std::array bad_entries = + { + { + json::object({{std::string("a\0b", 3), 1}}), + json::object({{"s", bad_utf8}}), + json::object({{bad_utf8, 1}}), + json::array({1, 2, bad_utf8}), + json::object({{"b", json::binary({1}, 0x100)}}), + json::object({{"d", json(json::value_t::discarded)}}), + json::array({json(json::value_t::discarded)}), + } + }; + for (const auto& bad : bad_entries) + { + for (std::size_t pos = 0; pos < 4; ++pos) + { + json document = json::object(); + json list = json::array(); + for (std::size_t i = 0; i < 4; ++i) + { + if (i == pos) + { + document["x" + std::to_string(i)] = bad; + list.push_back(bad); + } + document["n" + std::to_string(i)] = json::array({i, "str", json::object({{"k", i}})}); + list.push_back(i); + } + document["list"] = list; + for (const auto handler : handlers) + { + check_same(document, handler); + } + } + } + } +#endif +} From f022e3c59c21fff5be9a571070f43a45d978dd2f Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 1 Nov 2026 17:23:27 +0100 Subject: [PATCH 17/54] Binary readers and writers: faster UTF-8 checks of strings (#5982) Signed-off-by: Niels Lohmann --- .../nlohmann/detail/input/binary_reader.hpp | 24 +- include/nlohmann/detail/input/simd_scan.hpp | 72 ++++++ include/nlohmann/detail/input/string_scan.hpp | 58 +++++ .../nlohmann/detail/output/binary_writer.hpp | 89 ++++--- single_include/nlohmann/json.hpp | 243 +++++++++++++++--- tests/src/unit-guard-page.cpp | 1 + tests/src/unit-unicode.cpp | 38 +++ 7 files changed, 461 insertions(+), 64 deletions(-) diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index 190edb112..da5a0263e 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -4719,6 +4719,19 @@ class binary_reader return get_bytes(len, "string", result); } + /// whether @a s is valid UTF-8: ASCII eight bytes at a time, the rest + /// sequence by sequence (as the binary writers check strings) + static bool is_valid_utf8_string(const string_t& s, std::true_type /*is_byte*/) noexcept + { + return valid_utf8_string(reinterpret_cast(s.data()), s.size()); + } + + /// whether @a s, of a character type wider than a byte, is valid UTF-8 + static bool is_valid_utf8_string(const string_t& s, std::false_type /*is_byte*/) noexcept + { + return is_valid_utf8(s); + } + /*! @brief validate a decoded text string (value or object key) against @ref error_handler @@ -4738,7 +4751,16 @@ class binary_reader */ bool check_string_utf8(string_t& result, const char* context) { - if (error_handler == error_handler_t::keep || is_valid_utf8(result)) + return error_handler == error_handler_t::keep || check_string_utf8_handler(result, context); + } + + /// @ref check_string_utf8 for an error handler other than @ref + /// error_handler_t::keep; out of line, so that reading strings without a + /// check does not pay for the code of the check + JSON_HEDLEY_NEVER_INLINE + bool check_string_utf8_handler(string_t& result, const char* context) + { + if (is_valid_utf8_string(result, std::integral_constant())) { return true; } diff --git a/include/nlohmann/detail/input/simd_scan.hpp b/include/nlohmann/detail/input/simd_scan.hpp index 3abb911ee..6c34421fe 100644 --- a/include/nlohmann/detail/input/simd_scan.hpp +++ b/include/nlohmann/detail/input/simd_scan.hpp @@ -322,6 +322,78 @@ JSON_SCAN_SSSE3_TARGET JSON_HEDLEY_NEVER_INLINE inline bool vector_utf8_run(cons block_out = block; return false; } + +/*! +@brief skip 16-byte blocks of well-formed UTF-8 at p (a character boundary) + +Like vector_utf8_run, but quotes, backslashes, and control characters are +ordinary characters. Returns the start of the first block in which the check +found an error (which may belong to a sequence begun in the block before) or +where fewer than 16 bytes are left; a sequence may be cut at that point, so the +caller resumes at its start, one sequence at a time. +*/ +JSON_SCAN_SSSE3_TARGET JSON_HEDLEY_NEVER_INLINE inline const unsigned char* vector_utf8_valid_blocks(const unsigned char* block, const unsigned char* e) noexcept +{ + using lookup = utf8_lookup4<>; +#if JSON_SCAN_NEON + const uint8x16_t t1h = vld1q_u8(lookup::byte_1_high.data()); + const uint8x16_t t1l = vld1q_u8(lookup::byte_1_low.data()); + const uint8x16_t t2h = vld1q_u8(lookup::byte_2_high.data()); + uint8x16_t prev = vdupq_n_u8(0); + while (e - block >= 16) + { + const uint8x16_t in = vld1q_u8(block); + if (vmaxvq_u8(vorrq_u8(in, prev)) < 0x80) + { + // ASCII here, and no sequence from the block before to finish + prev = in; + block += 16; + continue; + } + const uint8x16_t prev1 = vextq_u8(prev, in, 15); + const uint8x16_t sc = vandq_u8(vandq_u8(vqtbl1q_u8(t1h, vshrq_n_u8(prev1, 4)), vqtbl1q_u8(t1l, vandq_u8(prev1, vdupq_n_u8(0x0F)))), vqtbl1q_u8(t2h, vshrq_n_u8(in, 4))); + const uint8x16_t must23 = vorrq_u8(vqsubq_u8(vextq_u8(prev, in, 14), vdupq_n_u8(0xE0 - 0x80)), vqsubq_u8(vextq_u8(prev, in, 13), vdupq_n_u8(0xF0 - 0x80))); + const uint8x16_t err = veorq_u8(vandq_u8(must23, vdupq_n_u8(0x80)), sc); + if (vmaxvq_u8(err) != 0) + { + break; + } + prev = in; + block += 16; + } +#else + const __m128i t1h = _mm_loadu_si128(static_cast(static_cast(lookup::byte_1_high.data()))); // NOLINT(bugprone-casting-through-void) + const __m128i t1l = _mm_loadu_si128(static_cast(static_cast(lookup::byte_1_low.data()))); // NOLINT(bugprone-casting-through-void) + const __m128i t2h = _mm_loadu_si128(static_cast(static_cast(lookup::byte_2_high.data()))); // NOLINT(bugprone-casting-through-void) + const __m128i nibble = _mm_set1_epi8(0x0F); + const __m128i zero = _mm_setzero_si128(); + __m128i prev = zero; + while (e - block >= 16) + { + const __m128i in = _mm_loadu_si128(static_cast(static_cast(block))); // NOLINT(bugprone-casting-through-void) + if (_mm_movemask_epi8(_mm_or_si128(in, prev)) == 0) + { + prev = in; + block += 16; + continue; + } + const __m128i prev1 = _mm_alignr_epi8(in, prev, 15); + const __m128i sc = _mm_and_si128(_mm_and_si128(_mm_shuffle_epi8(t1h, _mm_and_si128(_mm_srli_epi16(prev1, 4), nibble)), + _mm_shuffle_epi8(t1l, _mm_and_si128(prev1, nibble))), + _mm_shuffle_epi8(t2h, _mm_and_si128(_mm_srli_epi16(in, 4), nibble))); + const __m128i must23 = _mm_or_si128(_mm_subs_epu8(_mm_alignr_epi8(in, prev, 14), _mm_set1_epi8(0xE0 - 0x80)), + _mm_subs_epu8(_mm_alignr_epi8(in, prev, 13), _mm_set1_epi8(0xF0 - 0x80))); + const __m128i err = _mm_xor_si128(_mm_and_si128(must23, _mm_set1_epi8(static_cast(-128))), sc); + if (_mm_movemask_epi8(_mm_cmpeq_epi8(err, zero)) != 0xFFFF) + { + break; + } + prev = in; + block += 16; + } +#endif + return block; +} #endif } // namespace detail diff --git a/include/nlohmann/detail/input/string_scan.hpp b/include/nlohmann/detail/input/string_scan.hpp index daa9697e1..10e4cc900 100644 --- a/include/nlohmann/detail/input/string_scan.hpp +++ b/include/nlohmann/detail/input/string_scan.hpp @@ -264,6 +264,64 @@ inline std::size_t valid_utf8_prefix(const unsigned char* data, std::size_t n) n return pos; } +/*! +@brief whether [data, data+n) is valid UTF-8, for checking whole strings + +The same answer as valid_utf8_prefix(data, n) == n. Short ASCII strings, the +most common keys and values, are decided in one test, and long strings are +checked 16 bytes at a time where the vector check is available. +*/ +inline bool valid_utf8_string(const unsigned char* data, std::size_t n) noexcept +{ + constexpr std::uint64_t high = 0x8080808080808080ull; + if (n <= 16) + { + // two overlapping words cover 8 to 16 bytes, one partial word fewer + std::uint64_t first = 0; + std::uint64_t last = 0; + if (n >= 8) + { + std::memcpy(&first, data, sizeof(first)); + std::memcpy(&last, data + n - 8, sizeof(last)); + } + else if (n != 0) + { + std::memcpy(&first, data, n); + } + if (((first | last) & high) == 0) + { + return true; + } + } +#if JSON_SCAN_VECTOR_UTF8 + else if (n >= 32 +#if JSON_SCAN_SSSE3_DISPATCH + && cpu_has_ssse3() +#endif + ) + { + const unsigned char* block = vector_utf8_valid_blocks(data, data + n); + // the rest from the start of a sequence that may be cut at block + for (int i = 1; i <= 3 && block - i >= data; ++i) + { + const unsigned char c = block[-i]; + if (c < 0x80) + { + break; + } + if (c >= 0xC0) + { + block -= i; + break; + } + } + const auto checked = static_cast(block - data); + return checked + valid_utf8_prefix(block, n - checked) == n; + } +#endif + return valid_utf8_prefix(data, n) == n; +} + #if JSON_SCAN_VECTOR_UTF8 /// the end of scan_string_vector from block, where the vector loop stopped /// (ill-formed UTF-8, or fewer than 16 bytes left): one byte or sequence at a diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 9d21d2a7c..0c3c823ac 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -3307,6 +3307,25 @@ class binary_writer } } + /// write the head and the characters of @a value, at most + /// max_text_in_reservation of them, in one reservation + JSON_HEDLEY_ALWAYS_INLINE void write_cbor_string_reserved(const string_t& value) + { + CharType* const out = oa.reserve(value.size() + 2); + CharType* p = out; + if (value.size() <= 0x17) + { + *p++ = to_char_type(static_cast(0x60 + value.size())); + } + else + { + *p++ = to_char_type(0x78); + *p++ = to_char_type(static_cast(value.size())); + } + copy_characters(p, reinterpret_cast(value.data()), value.size()); + oa.commit(static_cast(p - out) + value.size()); + } + /*! @brief write a CBOR text string @@ -3328,19 +3347,7 @@ class binary_writer // the string is written as it is: head and characters in one reservation if (JSON_HEDLEY_LIKELY(value.size() <= max_text_in_reservation)) { - CharType* const out = oa.reserve(value.size() + 2); - CharType* p = out; - if (value.size() <= 0x17) - { - *p++ = to_char_type(static_cast(0x60 + value.size())); - } - else - { - *p++ = to_char_type(0x78); - *p++ = to_char_type(static_cast(value.size())); - } - copy_characters(p, reinterpret_cast(value.data()), value.size()); - oa.commit(static_cast(p - out) + value.size()); + write_cbor_string_reserved(value); return; } } @@ -3354,6 +3361,13 @@ class binary_writer JSON_HEDLEY_NEVER_INLINE void write_cbor_checked_string(const string_t& value, const BasicJsonType& context) { + if (error_handler == error_handler_t::strict && value.size() <= max_text_in_reservation) + { + check_utf8(value, context); + write_cbor_string_reserved(value); + return; + } + string_t storage; const string_t& sanitized = sanitize_utf8_for_write(value, context, storage); @@ -3368,6 +3382,27 @@ class binary_writer // MsgPack // ///////////// + /// write the head and the characters of @a value, at most + /// max_text_in_reservation of them, in one reservation + JSON_HEDLEY_ALWAYS_INLINE void write_msgpack_string_reserved(const string_t& value) + { + CharType* const out = oa.reserve(value.size() + 2); + CharType* p = out; + if (value.size() <= 31) + { + // fixstr + *p++ = to_char_type(static_cast(0xA0 | value.size())); + } + else + { + // str 8 + *p++ = to_char_type(0xD9); + *p++ = to_char_type(static_cast(value.size())); + } + copy_characters(p, reinterpret_cast(value.data()), value.size()); + oa.commit(static_cast(p - out) + value.size()); + } + /*! @brief write a MessagePack str @@ -3385,21 +3420,7 @@ class binary_writer // the string is written as it is: head and characters in one reservation if (JSON_HEDLEY_LIKELY(value.size() <= max_text_in_reservation)) { - CharType* const out = oa.reserve(value.size() + 2); - CharType* p = out; - if (value.size() <= 31) - { - // fixstr - *p++ = to_char_type(static_cast(0xA0 | value.size())); - } - else - { - // str 8 - *p++ = to_char_type(0xD9); - *p++ = to_char_type(static_cast(value.size())); - } - copy_characters(p, reinterpret_cast(value.data()), value.size()); - oa.commit(static_cast(p - out) + value.size()); + write_msgpack_string_reserved(value); return; } } @@ -3412,6 +3433,13 @@ class binary_writer JSON_HEDLEY_NEVER_INLINE void write_msgpack_checked_string(const string_t& value, const BasicJsonType& context) { + if (error_handler == error_handler_t::strict && value.size() <= max_text_in_reservation) + { + check_utf8(value, context); + write_msgpack_string_reserved(value); + return; + } + string_t storage; const string_t& sanitized = sanitize_utf8_for_write(value, context, storage); @@ -5021,10 +5049,9 @@ class binary_writer static void check_utf8(const string_t& s, const BasicJsonType& context) { const auto* data = reinterpret_cast(s.data()); - const std::size_t valid = valid_utf8_prefix(data, s.size()); - if (JSON_HEDLEY_UNLIKELY(valid != s.size())) + if (JSON_HEDLEY_UNLIKELY(!valid_utf8_string(data, s.size()))) { - throw_invalid_utf8(data, valid, context); + throw_invalid_utf8(data, valid_utf8_prefix(data, s.size()), context); } } diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 3c2670039..7b54e0a89 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -12379,6 +12379,78 @@ JSON_SCAN_SSSE3_TARGET JSON_HEDLEY_NEVER_INLINE inline bool vector_utf8_run(cons block_out = block; return false; } + +/*! +@brief skip 16-byte blocks of well-formed UTF-8 at p (a character boundary) + +Like vector_utf8_run, but quotes, backslashes, and control characters are +ordinary characters. Returns the start of the first block in which the check +found an error (which may belong to a sequence begun in the block before) or +where fewer than 16 bytes are left; a sequence may be cut at that point, so the +caller resumes at its start, one sequence at a time. +*/ +JSON_SCAN_SSSE3_TARGET JSON_HEDLEY_NEVER_INLINE inline const unsigned char* vector_utf8_valid_blocks(const unsigned char* block, const unsigned char* e) noexcept +{ + using lookup = utf8_lookup4<>; +#if JSON_SCAN_NEON + const uint8x16_t t1h = vld1q_u8(lookup::byte_1_high.data()); + const uint8x16_t t1l = vld1q_u8(lookup::byte_1_low.data()); + const uint8x16_t t2h = vld1q_u8(lookup::byte_2_high.data()); + uint8x16_t prev = vdupq_n_u8(0); + while (e - block >= 16) + { + const uint8x16_t in = vld1q_u8(block); + if (vmaxvq_u8(vorrq_u8(in, prev)) < 0x80) + { + // ASCII here, and no sequence from the block before to finish + prev = in; + block += 16; + continue; + } + const uint8x16_t prev1 = vextq_u8(prev, in, 15); + const uint8x16_t sc = vandq_u8(vandq_u8(vqtbl1q_u8(t1h, vshrq_n_u8(prev1, 4)), vqtbl1q_u8(t1l, vandq_u8(prev1, vdupq_n_u8(0x0F)))), vqtbl1q_u8(t2h, vshrq_n_u8(in, 4))); + const uint8x16_t must23 = vorrq_u8(vqsubq_u8(vextq_u8(prev, in, 14), vdupq_n_u8(0xE0 - 0x80)), vqsubq_u8(vextq_u8(prev, in, 13), vdupq_n_u8(0xF0 - 0x80))); + const uint8x16_t err = veorq_u8(vandq_u8(must23, vdupq_n_u8(0x80)), sc); + if (vmaxvq_u8(err) != 0) + { + break; + } + prev = in; + block += 16; + } +#else + const __m128i t1h = _mm_loadu_si128(static_cast(static_cast(lookup::byte_1_high.data()))); // NOLINT(bugprone-casting-through-void) + const __m128i t1l = _mm_loadu_si128(static_cast(static_cast(lookup::byte_1_low.data()))); // NOLINT(bugprone-casting-through-void) + const __m128i t2h = _mm_loadu_si128(static_cast(static_cast(lookup::byte_2_high.data()))); // NOLINT(bugprone-casting-through-void) + const __m128i nibble = _mm_set1_epi8(0x0F); + const __m128i zero = _mm_setzero_si128(); + __m128i prev = zero; + while (e - block >= 16) + { + const __m128i in = _mm_loadu_si128(static_cast(static_cast(block))); // NOLINT(bugprone-casting-through-void) + if (_mm_movemask_epi8(_mm_or_si128(in, prev)) == 0) + { + prev = in; + block += 16; + continue; + } + const __m128i prev1 = _mm_alignr_epi8(in, prev, 15); + const __m128i sc = _mm_and_si128(_mm_and_si128(_mm_shuffle_epi8(t1h, _mm_and_si128(_mm_srli_epi16(prev1, 4), nibble)), + _mm_shuffle_epi8(t1l, _mm_and_si128(prev1, nibble))), + _mm_shuffle_epi8(t2h, _mm_and_si128(_mm_srli_epi16(in, 4), nibble))); + const __m128i must23 = _mm_or_si128(_mm_subs_epu8(_mm_alignr_epi8(in, prev, 14), _mm_set1_epi8(0xE0 - 0x80)), + _mm_subs_epu8(_mm_alignr_epi8(in, prev, 13), _mm_set1_epi8(0xF0 - 0x80))); + const __m128i err = _mm_xor_si128(_mm_and_si128(must23, _mm_set1_epi8(static_cast(-128))), sc); + if (_mm_movemask_epi8(_mm_cmpeq_epi8(err, zero)) != 0xFFFF) + { + break; + } + prev = in; + block += 16; + } +#endif + return block; +} #endif } // namespace detail @@ -12634,6 +12706,64 @@ inline std::size_t valid_utf8_prefix(const unsigned char* data, std::size_t n) n return pos; } +/*! +@brief whether [data, data+n) is valid UTF-8, for checking whole strings + +The same answer as valid_utf8_prefix(data, n) == n. Short ASCII strings, the +most common keys and values, are decided in one test, and long strings are +checked 16 bytes at a time where the vector check is available. +*/ +inline bool valid_utf8_string(const unsigned char* data, std::size_t n) noexcept +{ + constexpr std::uint64_t high = 0x8080808080808080ull; + if (n <= 16) + { + // two overlapping words cover 8 to 16 bytes, one partial word fewer + std::uint64_t first = 0; + std::uint64_t last = 0; + if (n >= 8) + { + std::memcpy(&first, data, sizeof(first)); + std::memcpy(&last, data + n - 8, sizeof(last)); + } + else if (n != 0) + { + std::memcpy(&first, data, n); + } + if (((first | last) & high) == 0) + { + return true; + } + } +#if JSON_SCAN_VECTOR_UTF8 + else if (n >= 32 +#if JSON_SCAN_SSSE3_DISPATCH + && cpu_has_ssse3() +#endif + ) + { + const unsigned char* block = vector_utf8_valid_blocks(data, data + n); + // the rest from the start of a sequence that may be cut at block + for (int i = 1; i <= 3 && block - i >= data; ++i) + { + const unsigned char c = block[-i]; + if (c < 0x80) + { + break; + } + if (c >= 0xC0) + { + block -= i; + break; + } + } + const auto checked = static_cast(block - data); + return checked + valid_utf8_prefix(block, n - checked) == n; + } +#endif + return valid_utf8_prefix(data, n) == n; +} + #if JSON_SCAN_VECTOR_UTF8 /// the end of scan_string_vector from block, where the vector loop stopped /// (ill-formed UTF-8, or fewer than 16 bytes left): one byte or sequence at a @@ -22574,6 +22704,19 @@ class binary_reader return get_bytes(len, "string", result); } + /// whether @a s is valid UTF-8: ASCII eight bytes at a time, the rest + /// sequence by sequence (as the binary writers check strings) + static bool is_valid_utf8_string(const string_t& s, std::true_type /*is_byte*/) noexcept + { + return valid_utf8_string(reinterpret_cast(s.data()), s.size()); + } + + /// whether @a s, of a character type wider than a byte, is valid UTF-8 + static bool is_valid_utf8_string(const string_t& s, std::false_type /*is_byte*/) noexcept + { + return is_valid_utf8(s); + } + /*! @brief validate a decoded text string (value or object key) against @ref error_handler @@ -22593,7 +22736,16 @@ class binary_reader */ bool check_string_utf8(string_t& result, const char* context) { - if (error_handler == error_handler_t::keep || is_valid_utf8(result)) + return error_handler == error_handler_t::keep || check_string_utf8_handler(result, context); + } + + /// @ref check_string_utf8 for an error handler other than @ref + /// error_handler_t::keep; out of line, so that reading strings without a + /// check does not pay for the code of the check + JSON_HEDLEY_NEVER_INLINE + bool check_string_utf8_handler(string_t& result, const char* context) + { + if (is_valid_utf8_string(result, std::integral_constant())) { return true; } @@ -31067,6 +31219,25 @@ class binary_writer } } + /// write the head and the characters of @a value, at most + /// max_text_in_reservation of them, in one reservation + JSON_HEDLEY_ALWAYS_INLINE void write_cbor_string_reserved(const string_t& value) + { + CharType* const out = oa.reserve(value.size() + 2); + CharType* p = out; + if (value.size() <= 0x17) + { + *p++ = to_char_type(static_cast(0x60 + value.size())); + } + else + { + *p++ = to_char_type(0x78); + *p++ = to_char_type(static_cast(value.size())); + } + copy_characters(p, reinterpret_cast(value.data()), value.size()); + oa.commit(static_cast(p - out) + value.size()); + } + /*! @brief write a CBOR text string @@ -31088,19 +31259,7 @@ class binary_writer // the string is written as it is: head and characters in one reservation if (JSON_HEDLEY_LIKELY(value.size() <= max_text_in_reservation)) { - CharType* const out = oa.reserve(value.size() + 2); - CharType* p = out; - if (value.size() <= 0x17) - { - *p++ = to_char_type(static_cast(0x60 + value.size())); - } - else - { - *p++ = to_char_type(0x78); - *p++ = to_char_type(static_cast(value.size())); - } - copy_characters(p, reinterpret_cast(value.data()), value.size()); - oa.commit(static_cast(p - out) + value.size()); + write_cbor_string_reserved(value); return; } } @@ -31114,6 +31273,13 @@ class binary_writer JSON_HEDLEY_NEVER_INLINE void write_cbor_checked_string(const string_t& value, const BasicJsonType& context) { + if (error_handler == error_handler_t::strict && value.size() <= max_text_in_reservation) + { + check_utf8(value, context); + write_cbor_string_reserved(value); + return; + } + string_t storage; const string_t& sanitized = sanitize_utf8_for_write(value, context, storage); @@ -31128,6 +31294,27 @@ class binary_writer // MsgPack // ///////////// + /// write the head and the characters of @a value, at most + /// max_text_in_reservation of them, in one reservation + JSON_HEDLEY_ALWAYS_INLINE void write_msgpack_string_reserved(const string_t& value) + { + CharType* const out = oa.reserve(value.size() + 2); + CharType* p = out; + if (value.size() <= 31) + { + // fixstr + *p++ = to_char_type(static_cast(0xA0 | value.size())); + } + else + { + // str 8 + *p++ = to_char_type(0xD9); + *p++ = to_char_type(static_cast(value.size())); + } + copy_characters(p, reinterpret_cast(value.data()), value.size()); + oa.commit(static_cast(p - out) + value.size()); + } + /*! @brief write a MessagePack str @@ -31145,21 +31332,7 @@ class binary_writer // the string is written as it is: head and characters in one reservation if (JSON_HEDLEY_LIKELY(value.size() <= max_text_in_reservation)) { - CharType* const out = oa.reserve(value.size() + 2); - CharType* p = out; - if (value.size() <= 31) - { - // fixstr - *p++ = to_char_type(static_cast(0xA0 | value.size())); - } - else - { - // str 8 - *p++ = to_char_type(0xD9); - *p++ = to_char_type(static_cast(value.size())); - } - copy_characters(p, reinterpret_cast(value.data()), value.size()); - oa.commit(static_cast(p - out) + value.size()); + write_msgpack_string_reserved(value); return; } } @@ -31172,6 +31345,13 @@ class binary_writer JSON_HEDLEY_NEVER_INLINE void write_msgpack_checked_string(const string_t& value, const BasicJsonType& context) { + if (error_handler == error_handler_t::strict && value.size() <= max_text_in_reservation) + { + check_utf8(value, context); + write_msgpack_string_reserved(value); + return; + } + string_t storage; const string_t& sanitized = sanitize_utf8_for_write(value, context, storage); @@ -32781,10 +32961,9 @@ class binary_writer static void check_utf8(const string_t& s, const BasicJsonType& context) { const auto* data = reinterpret_cast(s.data()); - const std::size_t valid = valid_utf8_prefix(data, s.size()); - if (JSON_HEDLEY_UNLIKELY(valid != s.size())) + if (JSON_HEDLEY_UNLIKELY(!valid_utf8_string(data, s.size()))) { - throw_invalid_utf8(data, valid, context); + throw_invalid_utf8(data, valid_utf8_prefix(data, s.size()), context); } } diff --git a/tests/src/unit-guard-page.cpp b/tests/src/unit-guard-page.cpp index a4b12bb1a..24eaf33b7 100644 --- a/tests/src/unit-guard-page.cpp +++ b/tests/src/unit-guard-page.cpp @@ -403,6 +403,7 @@ TEST_CASE("guard page") CHECK(nlohmann::detail::find_string_special(p, n) <= n); CHECK(nlohmann::detail::find_ascii_copyable_run(p, n) <= n); CHECK(nlohmann::detail::valid_utf8_prefix(p, n) <= n); + CHECK(nlohmann::detail::valid_utf8_string(p, n) == (nlohmann::detail::valid_utf8_prefix(p, n) == n)); #if defined(JSON_USE_SIMDUTF) && defined(JSON_HAS_CPP_17) CHECK(nlohmann::detail::find_string_delimiter(p, n) <= n); #endif diff --git a/tests/src/unit-unicode.cpp b/tests/src/unit-unicode.cpp index 0207659ea..72c8f812f 100644 --- a/tests/src/unit-unicode.cpp +++ b/tests/src/unit-unicode.cpp @@ -13,6 +13,7 @@ using nlohmann::json; #include #include +#include #include #include "make_test_data_available.hpp" #include "test_utils.hpp" @@ -1631,3 +1632,40 @@ TEST_CASE("Unicode RFC 3629") } DOCTEST_CLANG_SUPPRESS_WARNING_POP + +TEST_CASE("UTF-8 check of whole strings") +{ + // valid_utf8_string() decides short strings with a word test and long ones + // 16 bytes at a time where a vector check is available; it must agree with + // the byte-wise check for every mix of ASCII, sequences of each length, + // ill-formed and truncated sequences, at every length and offset + const char* const pieces[] = // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + { + "a", "\x7F", "\xC3\xA9", "\xE2\x82\xAC", "\xE0\xA0\x80", "\xEF\xBF\xBF", "\xF0\x9F\x98\x80", "\xF4\x8F\xBF\xBF", + "\x80", "\xC0\xAF", "\xC3", "\xE2\x82", "\xED\xA0\x80", "\xF0\x9F", "\xF4\x90\x80\x80", "\xFF" + }; + // std::mt19937's sequence is fixed by the standard; no distribution is used + std::mt19937 rng(5); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed) + std::size_t valid = 0; + for (int i = 0; i < 100000; ++i) + { + std::string s; + const std::size_t length = rng() % 100; + const auto ascii = static_cast(rng() % 101); + while (s.size() < length) + { + // mostly well-formed pieces (the first eight), so that long valid + // strings are common + s += (rng() % 100 < ascii) ? std::string("x") : std::string(pieces[rng() % (rng() % 8 == 0 ? 16 : 8)]); + } + const auto* data = reinterpret_cast(s.data()); + const bool expected = nlohmann::detail::is_valid_utf8(s); + valid += expected ? 1 : 0; + CAPTURE(s) + CHECK(nlohmann::detail::valid_utf8_string(data, s.size()) == expected); + CHECK((nlohmann::detail::valid_utf8_prefix(data, s.size()) == s.size()) == expected); + } + // both outcomes are common enough to be tested + CHECK(valid > 20000); + CHECK(valid < 80000); +} From ddc34f546a652cd098dd78225d80f1909b921871 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 1 Nov 2026 17:25:26 +0100 Subject: [PATCH 18/54] json_view: exceptions name the path and the bytes of the value (#5974) Signed-off-by: Niels Lohmann --- docs/mkdocs/docs/api/basic_json_view/at.md | 7 +- docs/mkdocs/docs/api/basic_json_view/back.md | 6 +- docs/mkdocs/docs/api/basic_json_view/front.md | 6 +- docs/mkdocs/docs/api/basic_json_view/get.md | 15 +- .../docs/api/basic_json_view/get_string.md | 6 +- .../docs/api/basic_json_view/number_token.md | 6 +- .../docs/api/basic_json_view/operator[].md | 7 +- docs/mkdocs/docs/api/basic_json_view/value.md | 7 +- docs/mkdocs/docs/features/json_view.md | 34 +- include/nlohmann/detail/exceptions.hpp | 24 +- include/nlohmann/detail/macro_scope.hpp | 2 +- include/nlohmann/detail/view/edit.hpp | 22 +- include/nlohmann/detail/view/errors.hpp | 324 +++- include/nlohmann/detail/view/iterator.hpp | 5 +- include/nlohmann/detail/view/pointer.hpp | 9 +- include/nlohmann/detail/view/value.hpp | 6 +- include/nlohmann/json_view.hpp | 59 +- single_include/nlohmann/json.hpp | 26 +- single_include/nlohmann/json_view.hpp | 1332 ++++++++++------- tests/src/json_view_test_helpers.hpp | 22 + tests/src/unit-json_view.cpp | 110 +- tests/src/unit-json_view_edit.cpp | 22 +- tests/src/unit-json_view_from_json.cpp | 11 +- tests/src/unit-json_view_large.cpp | 2 +- tests/src/unit-json_view_positions.cpp | 226 +++ 25 files changed, 1600 insertions(+), 696 deletions(-) diff --git a/docs/mkdocs/docs/api/basic_json_view/at.md b/docs/mkdocs/docs/api/basic_json_view/at.md index 747b46299..7b810c4ed 100644 --- a/docs/mkdocs/docs/api/basic_json_view/at.md +++ b/docs/mkdocs/docs/api/basic_json_view/at.md @@ -68,9 +68,10 @@ Strong guarantee: there are no changes to the view or the document it refers to. - Throws [`out_of_range.404`](../../home/exceptions.md#jsonexceptionout_of_range404) if `ptr` cannot be resolved because a reference token is used on a primitive value. -None of these exceptions carry a [`JSON_DIAGNOSTICS`](../macros/json_diagnostics.md) path: the view has no -`BasicJsonType` value to point at, so the exception is created without one, even if `BasicJsonType` was built with -`JSON_DIAGNOSTICS` enabled. +As for `BasicJsonType`, with [`JSON_DIAGNOSTICS`](../macros/json_diagnostics.md) these exceptions name the JSON +Pointer path of the value within the document, and with +[`JSON_DIAGNOSTIC_POSITIONS`](../macros/json_diagnostic_positions.md) its bytes in the source text (when +[`materialize()`](materialize.md) sets them). ## Complexity diff --git a/docs/mkdocs/docs/api/basic_json_view/back.md b/docs/mkdocs/docs/api/basic_json_view/back.md index 473502f0a..f594928b0 100644 --- a/docs/mkdocs/docs/api/basic_json_view/back.md +++ b/docs/mkdocs/docs/api/basic_json_view/back.md @@ -20,9 +20,9 @@ Strong guarantee: there are no changes to the view or the document it refers to. Throws [`invalid_iterator.214`](../../home/exceptions.md#jsonexceptioninvalid_iterator214) if the view is [null](is_null.md) or [discarded](is_discarded.md), or if it is an empty array or object. -This exception does not carry a [`JSON_DIAGNOSTICS`](../macros/json_diagnostics.md) path: the view has no -`BasicJsonType` value to point at, so the exception is created without one, even if `BasicJsonType` was built with -`JSON_DIAGNOSTICS` enabled. +As for `BasicJsonType`, with [`JSON_DIAGNOSTICS`](../macros/json_diagnostics.md) the exception names the JSON Pointer +path of the value within the document, and with [`JSON_DIAGNOSTIC_POSITIONS`](../macros/json_diagnostic_positions.md) +its bytes in the source text (when [`materialize()`](materialize.md) sets them). ## Complexity diff --git a/docs/mkdocs/docs/api/basic_json_view/front.md b/docs/mkdocs/docs/api/basic_json_view/front.md index 4cefcbd33..da48fcf88 100644 --- a/docs/mkdocs/docs/api/basic_json_view/front.md +++ b/docs/mkdocs/docs/api/basic_json_view/front.md @@ -20,9 +20,9 @@ Strong guarantee: there are no changes to the view or the document it refers to. Throws [`invalid_iterator.214`](../../home/exceptions.md#jsonexceptioninvalid_iterator214) if the view is [null](is_null.md) or [discarded](is_discarded.md), or if it is an empty array or object. -This exception does not carry a [`JSON_DIAGNOSTICS`](../macros/json_diagnostics.md) path: the view has no -`BasicJsonType` value to point at, so the exception is created without one, even if `BasicJsonType` was built with -`JSON_DIAGNOSTICS` enabled. +As for `BasicJsonType`, with [`JSON_DIAGNOSTICS`](../macros/json_diagnostics.md) the exception names the JSON Pointer +path of the value within the document, and with [`JSON_DIAGNOSTIC_POSITIONS`](../macros/json_diagnostic_positions.md) +its bytes in the source text (when [`materialize()`](materialize.md) sets them). ## Complexity diff --git a/docs/mkdocs/docs/api/basic_json_view/get.md b/docs/mkdocs/docs/api/basic_json_view/get.md index bb9ac6f46..7ee7952bf 100644 --- a/docs/mkdocs/docs/api/basic_json_view/get.md +++ b/docs/mkdocs/docs/api/basic_json_view/get.md @@ -80,14 +80,13 @@ Strong guarantee: there are no changes to the view or the document it refers to. - For every other `T`: whatever [`materialize().get()`](../basic_json/get.md) throws — typically `type_error.302`, or whatever a user-provided `from_json()` throws. -None of the exceptions thrown directly by this function or by a `from_json()` that reads the view (the first four -bullets above) carry a -[`JSON_DIAGNOSTICS`](../macros/json_diagnostics.md) path: the view has no `BasicJsonType` value to point at. An -exception thrown while converting through `materialize()` (the last bullet) is different: it is thrown by a real -`BasicJsonType` value, so it **does** carry a `JSON_DIAGNOSTICS` path if `BasicJsonType` was built with it enabled. -That path is relative to the root of the view's subtree, because `materialize()` builds only the subtree (with parent -pointers): it is empty if the failing value is the subtree's root itself, and it never is the path within the whole -document. +The exceptions thrown directly by this function or by a `from_json()` that reads the view (the first four bullets +above) carry the context of those of `BasicJsonType`: with [`JSON_DIAGNOSTICS`](../macros/json_diagnostics.md) the +JSON Pointer path of the value within the document, and with +[`JSON_DIAGNOSTIC_POSITIONS`](../macros/json_diagnostic_positions.md) its bytes in the source text. An exception +thrown while converting through `materialize()` (the last bullet) is thrown by the materialized `BasicJsonType` value +instead: its path is relative to the root of the view's subtree, because `materialize()` builds only the subtree +(with parent pointers), so it is empty if the failing value is the subtree's root itself. ## Complexity diff --git a/docs/mkdocs/docs/api/basic_json_view/get_string.md b/docs/mkdocs/docs/api/basic_json_view/get_string.md index 7539c540d..b7d4d4203 100644 --- a/docs/mkdocs/docs/api/basic_json_view/get_string.md +++ b/docs/mkdocs/docs/api/basic_json_view/get_string.md @@ -22,9 +22,9 @@ Strong guarantee: there are no changes to the view or the document it refers to. Throws [`type_error.302`](../../home/exceptions.md#jsonexceptiontype_error302) if the value is not a string; example: `"type must be string, but is array"`. -This exception does not carry a [`JSON_DIAGNOSTICS`](../macros/json_diagnostics.md) path: the view has no -`BasicJsonType` value to point at, so the exception is created without one, even if `BasicJsonType` was built with -`JSON_DIAGNOSTICS` enabled. +As for `BasicJsonType`, with [`JSON_DIAGNOSTICS`](../macros/json_diagnostics.md) the exception names the JSON Pointer +path of the value within the document, and with [`JSON_DIAGNOSTIC_POSITIONS`](../macros/json_diagnostic_positions.md) +its bytes in the source text (when [`materialize()`](materialize.md) sets them). ## Complexity diff --git a/docs/mkdocs/docs/api/basic_json_view/number_token.md b/docs/mkdocs/docs/api/basic_json_view/number_token.md index 97d7a2ed2..6451cb1fa 100644 --- a/docs/mkdocs/docs/api/basic_json_view/number_token.md +++ b/docs/mkdocs/docs/api/basic_json_view/number_token.md @@ -22,9 +22,9 @@ Strong guarantee: there are no changes to the view or the document it refers to. Throws [`type_error.302`](../../home/exceptions.md#jsonexceptiontype_error302) if the value is not a number; example: `"type must be number, but is string"`. -This exception does not carry a [`JSON_DIAGNOSTICS`](../macros/json_diagnostics.md) path: the view has no -`BasicJsonType` value to point at, so the exception is created without one, even if `BasicJsonType` was built with -`JSON_DIAGNOSTICS` enabled. +As for `BasicJsonType`, with [`JSON_DIAGNOSTICS`](../macros/json_diagnostics.md) the exception names the JSON Pointer +path of the value within the document, and with [`JSON_DIAGNOSTIC_POSITIONS`](../macros/json_diagnostic_positions.md) +its bytes in the source text (when [`materialize()`](materialize.md) sets them). ## Complexity diff --git a/docs/mkdocs/docs/api/basic_json_view/operator[].md b/docs/mkdocs/docs/api/basic_json_view/operator[].md index 9c09a807b..a04bfd3b8 100644 --- a/docs/mkdocs/docs/api/basic_json_view/operator[].md +++ b/docs/mkdocs/docs/api/basic_json_view/operator[].md @@ -73,9 +73,10 @@ Strong guarantee: there are no changes to the view or the document it refers to. - [`parse_error.109`](../../home/exceptions.md#jsonexceptionparse_error109) if an array index in `ptr` is not a number. -None of these exceptions carry a [`JSON_DIAGNOSTICS`](../macros/json_diagnostics.md) path: the view has no -`BasicJsonType` value to point at, so the exception is created without one, even if `BasicJsonType` was built with -`JSON_DIAGNOSTICS` enabled. +As for `BasicJsonType`, with [`JSON_DIAGNOSTICS`](../macros/json_diagnostics.md) these exceptions name the JSON +Pointer path of the value within the document, and with +[`JSON_DIAGNOSTIC_POSITIONS`](../macros/json_diagnostic_positions.md) its bytes in the source text (when +[`materialize()`](materialize.md) sets them). ## Complexity diff --git a/docs/mkdocs/docs/api/basic_json_view/value.md b/docs/mkdocs/docs/api/basic_json_view/value.md index 7159325b2..ebce7b4bf 100644 --- a/docs/mkdocs/docs/api/basic_json_view/value.md +++ b/docs/mkdocs/docs/api/basic_json_view/value.md @@ -61,9 +61,10 @@ Strong guarantee: there are no changes to the view or the document it refers to. `default_value` instead of throwing, exactly as [`BasicJsonType::value`](../basic_json/value.md) catches `out_of_range` and returns `default_value`. -None of these exceptions carry a [`JSON_DIAGNOSTICS`](../macros/json_diagnostics.md) path: the view has no -`BasicJsonType` value to point at, so the exception is created without one, even if `BasicJsonType` was built with -`JSON_DIAGNOSTICS` enabled. +As for `BasicJsonType`, with [`JSON_DIAGNOSTICS`](../macros/json_diagnostics.md) these exceptions name the JSON +Pointer path of the value within the document, and with +[`JSON_DIAGNOSTIC_POSITIONS`](../macros/json_diagnostic_positions.md) its bytes in the source text (when +[`materialize()`](materialize.md) sets them). ## Complexity diff --git a/docs/mkdocs/docs/features/json_view.md b/docs/mkdocs/docs/features/json_view.md index 08529fd7d..44bf581a5 100644 --- a/docs/mkdocs/docs/features/json_view.md +++ b/docs/mkdocs/docs/features/json_view.md @@ -134,9 +134,21 @@ document: `#!cpp auto v = json_document::parse(text).root();` does not compile. - **Macros.** [`JSON_STRICT_NUL_HANDLING`](../api/macros/json_strict_nul_handling.md) and [`JSON_NOEXCEPTION`](../api/macros/json_noexception.md)/[`JSON_THROW_USER`](../api/macros/json_throw_user.md) behave the same way they do for ``. These are the macros that change what the parser accepts or - how it reports errors. The others either do not apply ([`JSON_DIAGNOSTICS`](../api/macros/json_diagnostics.md) - needs parent pointers in `basic_json` values) or only affect the values `materialize()` produces, through - `BasicJsonType`. + how it reports errors. Two more macros add context to the exceptions of views, as they do for those of + `#!cpp basic_json`: + + - With [`JSON_DIAGNOSTICS`](../api/macros/json_diagnostics.md), an exception names the JSON Pointer path of the + value within the document. + - With [`JSON_DIAGNOSTIC_POSITIONS`](../api/macros/json_diagnostic_positions.md), an exception names the bytes of + the value in the source text. + + A view has neither parent pointers nor stored positions, so both are found from the document's index when the + exception is thrown. This takes time linear in the size of the document. One exception has neither: + `key()` of an iterator over an array or a primitive value + ([`invalid_iterator.207`](../home/exceptions.md#jsonexceptioninvalid_iterator207)), because a view's iterator does + not know the value it iterates over. + + The other macros only affect the values `materialize()` produces, through `BasicJsonType`. ## What is different @@ -167,10 +179,6 @@ document: `#!cpp auto v = json_document::parse(text).root();` does not compile. [`at`](../api/basic_json_view/at.md) throws for every missing value. - **Duplicate keys: lookups find the first member, `parse()` keeps the last.** See [Duplicate keys](#duplicate-keys) below. -- **No [`JSON_DIAGNOSTICS`](../api/macros/json_diagnostics.md) path.** Exceptions thrown by `basic_json_view`'s own - element access and lookup functions never carry the JSON Pointer path `JSON_DIAGNOSTICS` would otherwise add: the - view has no `basic_json` value to point at, so the exception is created without one, regardless of how - `BasicJsonType` was built. - **Ordering comparisons are not provided** by `basic_json_view` — there is no `#!cpp operator<`. [`operator==`](../api/basic_json_view/operator_eq.md) and [`operator!=`](../api/basic_json_view/operator_ne.md) are provided, though: two views, or a view and a `BasicJsonType` value, compare equal exactly when @@ -357,14 +365,10 @@ Its body may only use what views and `basic_json` have in common: the type check a view does not have; such a type is converted through `materialize()`. A `from_json()` for `json` only is converted through `materialize()` as well. -Two things differ when a `from_json()` reads the view: - -- **Duplicate keys.** Members looked up by name (`at()`, `value()`, ... and thus the macros) find the **first** member - with a repeated key, see [Duplicate keys](#duplicate-keys); `#!cpp materialize().get()` would see the last one. - Containers built from an object, such as `#!cpp std::map`, keep the **last** value either way, as - [`parse()`](../api/basic_json/parse.md) does. -- **[`JSON_DIAGNOSTICS`](../api/macros/json_diagnostics.md).** Exceptions thrown by the view carry no path to the - value. +One thing differs when a `from_json()` reads the view: members looked up by name (`at()`, `value()`, ... and thus +the macros) find the **first** member with a repeated key, see [Duplicate keys](#duplicate-keys); +`#!cpp materialize().get()` would see the last one. Containers built from an object, such as `#!cpp std::map`, +keep the **last** value either way, as [`parse()`](../api/basic_json/parse.md) does. `to_json()` is unaffected: it writes `basic_json` values, never views. diff --git a/include/nlohmann/detail/exceptions.hpp b/include/nlohmann/detail/exceptions.hpp index ee2c0feec..144bb770c 100644 --- a/include/nlohmann/detail/exceptions.hpp +++ b/include/nlohmann/detail/exceptions.hpp @@ -431,6 +431,28 @@ std::nullptr_t exception_context(const View& /*unused*/) noexcept return nullptr; } +namespace view +{ +// the path and bytes of the value of a view (defined in ) +template +std::string diagnostics_of(const View& v); +} // namespace view + +// what goes in front of the message of an exception about j: nothing for a +// basic_json value (exception_context() passes the value itself), the context +// basic_json would add (JSON_DIAGNOSTICS, JSON_DIAGNOSTIC_POSITIONS) for a view +template::value, int> = 0> +const char* exception_prefix(const BasicJsonType& /*unused*/) noexcept +{ + return ""; +} + +template::value, int> = 0> +std::string exception_prefix(const View& v) +{ + return view::diagnostics_of(v); +} + /*! @brief throws because @a j does not have the type a conversion expects @param[in] expected the expected type(s), e.g. "array" or "binary or array" @@ -442,7 +464,7 @@ JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_type_must_be(const cha { static_cast(expected); // unused when JSON_NOEXCEPTION is defined static_cast(j); - JSON_THROW(type_error::create(exception_id::type_mismatch, concat("type must be ", expected, ", but is ", j.type_name()), exception_context(j))); + JSON_THROW(type_error::create(exception_id::type_mismatch, concat(exception_prefix(j), "type must be ", expected, ", but is ", j.type_name()), exception_context(j))); } /*! diff --git a/include/nlohmann/detail/macro_scope.hpp b/include/nlohmann/detail/macro_scope.hpp index 65394b504..208c1c3ce 100644 --- a/include/nlohmann/detail/macro_scope.hpp +++ b/include/nlohmann/detail/macro_scope.hpp @@ -347,7 +347,7 @@ return ej_pair.second == j; \ }); \ if (it != std::end(m)) e = it->first; \ - else ::nlohmann::detail::templated_json_throw(nlohmann::detail::out_of_range::create(nlohmann::detail::exception_id::value_out_of_range, nlohmann::detail::concat("enum value out of range for " #ENUM_TYPE ": ", nlohmann::detail::from_json_dump(j)), nlohmann::detail::exception_context(j))); \ + else ::nlohmann::detail::templated_json_throw(nlohmann::detail::out_of_range::create(nlohmann::detail::exception_id::value_out_of_range, nlohmann::detail::concat(nlohmann::detail::exception_prefix(j), "enum value out of range for " #ENUM_TYPE ": ", nlohmann::detail::from_json_dump(j)), nlohmann::detail::exception_context(j))); \ } // Ugly macros to avoid uglier copy-paste when specializing basic_json. They diff --git a/include/nlohmann/detail/view/edit.hpp b/include/nlohmann/detail/view/edit.hpp index c542c1141..b572b4848 100644 --- a/include/nlohmann/detail/view/edit.hpp +++ b/include/nlohmann/detail/view/edit.hpp @@ -157,7 +157,7 @@ class editor node* const o = own(object); if (o->kind != node_kind::object && o->kind != node_kind::null) { - throw_type_error(exception_id::subscript_wrong_type, "cannot use operator[] with a string argument with ", object.type_name()); + throw_type_error(site_of(object), exception_id::subscript_wrong_type, "cannot use operator[] with a string argument with ", object.type_name()); } check_utf8(key.data(), key.size()); const encoded e = encode(std::forward(value)); @@ -208,9 +208,9 @@ class editor node* const a = own(array); if (a->kind != node_kind::array) { - throw_type_error(exception_id::subscript_wrong_type, "cannot use operator[] with a numeric argument with ", array.type_name()); + throw_type_error(site_of(array), exception_id::subscript_wrong_type, "cannot use operator[] with a numeric argument with ", array.type_name()); } - check_index(idx, a->len); + check_index(a, idx, a->len); const encoded e = encode(std::forward(value)); node* const slot = const_cast(nav::value(element_at(m_doc, a, idx))); // NOLINT(cppcoreguidelines-pro-type-const-cast) assign(slot, e, a, true); @@ -225,7 +225,7 @@ class editor node* const a = own(array); if (a->kind != node_kind::array && a->kind != node_kind::null) { - throw_type_error(exception_id::push_back_wrong_type, "cannot use push_back() with ", array.type_name()); + throw_type_error(site_of(array), exception_id::push_back_wrong_type, "cannot use push_back() with ", array.type_name()); } const encoded e = encode(std::forward(value)); if (a->kind == node_kind::null) @@ -249,9 +249,9 @@ class editor node* const a = own(array); if (a->kind != node_kind::array) { - throw_type_error(exception_id::insert_wrong_type, "cannot use insert() with ", array.type_name()); + throw_type_error(site_of(array), exception_id::insert_wrong_type, "cannot use insert() with ", array.type_name()); } - check_index(idx, a->len + 1); + check_index(a, idx, a->len + 1); const encoded e = encode(std::forward(value)); node* const slot = new_slot(e); node* const h = block_of(m_doc, a, 1); @@ -269,7 +269,7 @@ class editor node* const o = own(object); if (o->kind != node_kind::object) { - throw_type_error(exception_id::erase_wrong_type, "cannot use erase() with ", object.type_name()); + throw_type_error(site_of(object), exception_id::erase_wrong_type, "cannot use erase() with ", object.type_name()); } for (const node* k = nav::first(m_doc, o), *end = nav::end(m_doc, o); k != end; k = document_data::after(k + 1)) { @@ -287,9 +287,9 @@ class editor node* const a = own(array); if (a->kind != node_kind::array) { - throw_type_error(exception_id::erase_wrong_type, "cannot use erase() with ", array.type_name()); + throw_type_error(site_of(array), exception_id::erase_wrong_type, "cannot use erase() with ", array.type_name()); } - check_index(idx, a->len); + check_index(a, idx, a->len); node* const h = block_of(m_doc, a, 0); std::memmove(h + 1 + idx, h + 2 + idx, (h->next - 2 - idx) * sizeof(node)); --h->next; @@ -316,11 +316,11 @@ class editor return const_cast(v.m_node); // NOLINT(cppcoreguidelines-pro-type-const-cast): the nodes belong to this document } - static void check_index(std::size_t idx, std::size_t limit) + void check_index(const node* array, std::size_t idx, std::size_t limit) const { if (idx >= limit) { - throw_array_index_out_of_range(idx); + throw_array_index_out_of_range(error_site(&m_doc, array), idx); } } diff --git a/include/nlohmann/detail/view/errors.hpp b/include/nlohmann/detail/view/errors.hpp index 590bb2edb..eeab3bbd5 100644 --- a/include/nlohmann/detail/view/errors.hpp +++ b/include/nlohmann/detail/view/errors.hpp @@ -10,11 +10,15 @@ #include // min #include // size_t -#include // string +#include // string, to_string +#include // vector #include #include +#include #include +#include +#include NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail @@ -22,8 +26,293 @@ namespace detail namespace view { +/*! +@brief the value an exception is about + +basic_json passes the value to the exception, which then names its path +(JSON_DIAGNOSTICS) and its bytes in the source (JSON_DIAGNOSTIC_POSITIONS). A +view's value has no parent pointer and no stored positions: both are found +from the document and the node, when the exception is thrown (so only the +error path pays for them). Without the two macros, a site is empty. +*/ +class error_site +{ + public: + error_site() noexcept = default; + +#if JSON_DIAGNOSTICS || JSON_DIAGNOSTIC_POSITIONS + error_site(const document_data* d, const node* n) noexcept + : m_doc(d), m_node(n) + {} + + const document_data* doc() const noexcept + { + return m_doc; + } + const node* value() const noexcept + { + return m_node; + } + + private: + const document_data* m_doc = nullptr; + const node* m_node = nullptr; +#else + error_site(const document_data* /*d*/, const node* /*n*/) noexcept {} +#endif +}; + +/// the site of a view's value (the views make this a friend) +struct view_access +{ + template + static error_site site(const View& v) noexcept + { + return {v.m_doc, v.m_node}; + } +}; + +template +error_site site_of(const View& v) noexcept +{ + static_assert(is_json_view::value, "the site of a value is only defined for views"); + return view_access::site(v); +} + +#if JSON_DIAGNOSTICS +/*! +@brief the JSON pointer tokens from the root of document @a d to node @a target + +Nodes have no parent index, so the path is found by descending from the root: +before the first edit, only into the subtree that contains the target (a +container spans the next `next` nodes); after edits, elements can live in +other sequences, so every container is searched. Iterative (the nesting depth +is limited by memory only). False if the target is not found (then tokens is +unspecified). +*/ +template +bool value_path(const document_data& d, const node* target, std::vector& tokens) +{ + using nav = navigation; + struct frame + { + const node* pos; + const node* end; + std::size_t index; + bool object; + }; + const node* const root = d.nodes; + if (target == root) + { + return true; + } + if (!is_container(*root)) + { + return false; + } + std::vector stack; + stack.push_back(frame{nav::first(d, root), nav::end(d, root), 0, root->kind == node_kind::object}); + // (tokens[i] leads from stack[i] to stack[i + 1], or to the target) + for (;;) + { + // a copy: pop_back() and push_back() below change the stack + const frame f = stack.back(); + if (f.pos == f.end) + { + stack.pop_back(); + if (stack.empty()) + { + return false; + } + tokens.pop_back(); + continue; + } + const node* entry = f.pos; + std::string token; + if (f.object) + { + // (as basic_json: the key up to a null byte) + token.assign(d.str(*entry), entry->len); + const std::size_t nul = token.find('\0'); + if (nul != std::string::npos) + { + token.resize(nul); + } + ++entry; + } + else + { + token = std::to_string(f.index); + } + const node* const child = nav::value(entry); + if (child == target) + { + tokens.push_back(token); + return true; + } + stack.back().pos = document_data::after(entry); + ++stack.back().index; + if (is_container(*child) && (nav::editable() || (target > child && target < child + child->next))) + { + tokens.push_back(token); + stack.push_back(frame{nav::first(d, child), nav::end(d, child), 0, child->kind == node_kind::object}); + } + } +} +#endif + +#if JSON_DIAGNOSTIC_POSITIONS +/*! +@brief the bytes [start, end) of node @a n in the source, as materialize() sets them + +Scalars as position_source::set_scalar; an array or object from its opening +bracket to its closing bracket, which is found by following the chain of last +elements to a scalar (or an empty container) and scanning from there to the +closing brackets. None (npos) for values written by edits, and for containers +whose last element has none, as with materialize(). +*/ +template +void value_range(const document_data& d, const node* n, std::size_t& start, std::size_t& end) +{ + using nav = navigation; + position_source p; + start = std::string::npos; + end = std::string::npos; + if (!is_container(*n)) + { + p.set_scalar(d, *n); + start = p.start; + end = p.end; + return; + } + if ((n->flags & node_flags::is_new) != 0 || n->off >= d.size) + { + return; + } + // the open brackets: n, then the last element as long as it is a non-empty container + std::size_t closing = 0; + std::size_t pos = std::string::npos; + for (const node* c = n;;) + { + if (!is_container(*c)) + { + p.set_scalar(d, *c); + pos = p.end; + break; + } + if ((c->flags & node_flags::is_new) != 0 || c->off >= d.size) + { + return; + } + ++closing; + const node* const first = nav::first(d, c); + const node* const last_end = nav::end(d, c); + if (first == last_end) + { + pos = std::size_t{c->off} + 1; + break; + } + const std::size_t value_offset = c->kind == node_kind::object ? 1 : 0; + const node* last = first; + for (const node* e = document_data::after(first + value_offset); e != last_end; e = document_data::after(e + value_offset)) + { + last = e; + } + c = nav::value(last + value_offset); + } + for (; closing != 0 && pos != std::string::npos; --closing) + { + pos = position_source::container_end(d.base[0], d.size, pos); + } + if (pos != std::string::npos) + { + start = n->off; + end = pos; + } +} +#endif + +/// the context basic_json puts in front of the message of an exception about +/// a value: "(/path) " with JSON_DIAGNOSTICS (none for the root), then +/// "(bytes a-b) " with JSON_DIAGNOSTIC_POSITIONS (with JSON_DIAGNOSTICS, only +/// if there is a path) +/// +/// This and the throw helpers that take an error_site are templates, so that +/// the path and range code is instantiated only where an exception can be +/// thrown: instantiated in every translation unit (as a non-template inline +/// function makes it), it changed GCC's early inlining of unrelated view +/// functions, so that GCC 13 no longer inlined a recursive walk over a view +/// into itself (2.7 times the instructions). +#if JSON_DIAGNOSTICS || JSON_DIAGNOSTIC_POSITIONS +template +std::string diagnostics(const error_site site) +{ + if (site.doc() == nullptr || site.value() == nullptr) + { + return ""; + } + const document_data& d = *site.doc(); + const bool edited = d.edits != nullptr; + std::string context; +#if JSON_DIAGNOSTICS + std::vector tokens; + const bool found = edited ? value_path(d, site.value(), tokens) : value_path(d, site.value(), tokens); + if (!found || tokens.empty()) + { + // context is still empty; returning it keeps the copy elided (-Wnrvo) + return context; + } + std::string path; + for (const std::string& token : tokens) + { + path.push_back('/'); + path += detail::escape(token); + } + context = concat('(', path, ") "); +#endif +#if JSON_DIAGNOSTIC_POSITIONS + std::size_t start = 0; + std::size_t end = 0; + if (edited) + { + value_range(d, site.value(), start, end); + } + else + { + value_range(d, site.value(), start, end); + } + if (start != std::string::npos && end != std::string::npos) + { + context += concat("(bytes ", std::to_string(start), "-", std::to_string(end), ") "); + } +#endif + return context; +} +#else +template +std::string diagnostics(const error_site /*site*/) +{ + return ""; +} +#endif + +/// the context of an exception about view @a v (see exception_prefix()) +template +std::string diagnostics_of(const View& v) +{ + return diagnostics<>(site_of(v)); +} + // Exceptions are thrown out of line, so that the accessors that may throw stay -// small enough to be inlined. +// small enough to be inlined. The overloads with an error_site put its context +// in front of the message. +// +// The overloads without one are for exceptions that are not about a value of a +// document: limits and invalid input of the editing and image functions, a +// discarded view (it has no node), a view of another document (its path would +// be one in that document), and those that basic_json creates without a +// context too (the errors of json_pointer, invalid UTF-8). The one exception +// about a value that has no site is key(), see view_iterator::key(). [[noreturn]] NLOHMANN_VIEW_NOINLINE inline void throw_type_error(exception_id id, const char* prefix, const char* type) { @@ -45,6 +334,24 @@ namespace view NLOHMANN_VIEW_THROW(invalid_iterator::create(id, msg, nullptr)); } +template +[[noreturn]] NLOHMANN_VIEW_NOINLINE void throw_type_error(const error_site site, exception_id id, const char* prefix, const char* type) +{ + NLOHMANN_VIEW_THROW(type_error::create(id, concat(diagnostics<>(site), prefix, type), nullptr)); +} + +template +[[noreturn]] NLOHMANN_VIEW_NOINLINE void throw_out_of_range(const error_site site, exception_id id, const std::string& msg) +{ + NLOHMANN_VIEW_THROW(out_of_range::create(id, concat(diagnostics<>(site), msg), nullptr)); +} + +template +[[noreturn]] NLOHMANN_VIEW_NOINLINE void throw_invalid_iterator(const error_site site, exception_id id, const char* msg) +{ + NLOHMANN_VIEW_THROW(invalid_iterator::create(id, concat(diagnostics<>(site), msg), nullptr)); +} + /// a parse error without a position (as those of json_pointer) [[noreturn]] NLOHMANN_VIEW_NOINLINE inline void throw_parse_error(exception_id id, const std::string& msg) { @@ -53,22 +360,23 @@ namespace view /// out_of_range.401: the array index @a idx is not below the size template -[[noreturn]] NLOHMANN_VIEW_NOINLINE void throw_array_index_out_of_range(const Index idx) +[[noreturn]] NLOHMANN_VIEW_NOINLINE void throw_array_index_out_of_range(const error_site site, const Index idx) { - throw_out_of_range(exception_id::array_index_out_of_range, concat("array index ", std::to_string(idx), " is out of range")); + throw_out_of_range(site, exception_id::array_index_out_of_range, concat("array index ", std::to_string(idx), " is out of range")); } /// out_of_range.402: the JSON Pointer token '-' of an array of @a size elements -[[noreturn]] NLOHMANN_VIEW_NOINLINE inline void throw_array_index_past_end(std::size_t size) +template +[[noreturn]] NLOHMANN_VIEW_NOINLINE void throw_array_index_past_end(const error_site site, std::size_t size) { - throw_out_of_range(exception_id::pointer_past_the_end_index, concat("array index '-' (", std::to_string(size), ") is out of range")); + throw_out_of_range(site, exception_id::pointer_past_the_end_index, concat("array index '-' (", std::to_string(size), ") is out of range")); } /// out_of_range.403: the object has no member @a key template -[[noreturn]] NLOHMANN_VIEW_NOINLINE void throw_key_not_found(const Key& key) +[[noreturn]] NLOHMANN_VIEW_NOINLINE void throw_key_not_found(const error_site site, const Key& key) { - throw_out_of_range(exception_id::key_not_found, concat("key '", key, "' not found")); + throw_out_of_range(site, exception_id::key_not_found, concat("key '", key, "' not found")); } /// SAX handler of the error replay: accepts every event without storing it, diff --git a/include/nlohmann/detail/view/iterator.hpp b/include/nlohmann/detail/view/iterator.hpp index 9d8246210..fa0cc744e 100644 --- a/include/nlohmann/detail/view/iterator.hpp +++ b/include/nlohmann/detail/view/iterator.hpp @@ -105,7 +105,10 @@ class view_iterator } /// the key of the current object member; throws invalid_iterator.207 for - /// other iterators, like basic_json's iterators + /// other iterators, like basic_json's iterators (but without the context + /// JSON_DIAGNOSTICS and JSON_DIAGNOSTIC_POSITIONS add: the iterator does not + /// know the value it iterates over, and keeping it would make iterators + /// larger, which keeps GCC from inlining recursive walks) string_view_t key() const { if (NLOHMANN_VIEW_UNLIKELY(m_value_offset == 0)) diff --git a/include/nlohmann/detail/view/pointer.hpp b/include/nlohmann/detail/view/pointer.hpp index be5aa14de..c17561134 100644 --- a/include/nlohmann/detail/view/pointer.hpp +++ b/include/nlohmann/detail/view/pointer.hpp @@ -44,6 +44,7 @@ index_status array_index(const StringType& s, std::size_t& idx) noexcept } /// throws the exception json_pointer::array_index throws for this status +/// (without a context, as json_pointer::array_index) template [[noreturn]] NLOHMANN_VIEW_NOINLINE void throw_array_index_error(index_status status, const StringType& s) { @@ -83,7 +84,7 @@ View resolve_pointer(View cur, const Tokens& tokens, pointer_mode mode) { if (mode == pointer_mode::checked) { - throw_key_not_found(token); + throw_key_not_found(site_of(cur), token); } return View(); } @@ -95,7 +96,7 @@ View resolve_pointer(View cur, const Tokens& tokens, pointer_mode mode) { if (throwing) { - throw_array_index_past_end(cur.size()); + throw_array_index_past_end(site_of(cur), cur.size()); } return View(); } @@ -114,7 +115,7 @@ View resolve_pointer(View cur, const Tokens& tokens, pointer_mode mode) { if (mode == pointer_mode::checked) { - throw_array_index_out_of_range(idx); + throw_array_index_out_of_range(site_of(cur), idx); } return View(); } @@ -124,7 +125,7 @@ View resolve_pointer(View cur, const Tokens& tokens, pointer_mode mode) { if (throwing) { - throw_out_of_range(exception_id::pointer_unresolved, concat("unresolved reference token '", token, "'")); + throw_out_of_range(site_of(cur), exception_id::pointer_unresolved, concat("unresolved reference token '", token, "'")); } return View(); } diff --git a/include/nlohmann/detail/view/value.hpp b/include/nlohmann/detail/view/value.hpp index 4b54a7785..ffdcc3605 100644 --- a/include/nlohmann/detail/view/value.hpp +++ b/include/nlohmann/detail/view/value.hpp @@ -60,7 +60,7 @@ NLOHMANN_VIEW_ALWAYS_INLINE T arithmetic_value(const document_data& d, const nod case value_t::binary: case value_t::discarded: default: - throw_type_error(exception_id::type_mismatch, "type must be number, but is ", value_type_name(static_cast(n.kind))); + throw_type_error(error_site(&d, &n), exception_id::type_mismatch, "type must be number, but is ", value_type_name(static_cast(n.kind))); } } @@ -70,7 +70,7 @@ std::vector vector_value(const View& v) { if (NLOHMANN_VIEW_UNLIKELY(!v.is_array())) { - throw_type_error(exception_id::type_mismatch, "type must be array, but is ", v.type_name()); + throw_type_error(site_of(v), exception_id::type_mismatch, "type must be array, but is ", v.type_name()); } std::vector r; r.reserve(v.size()); @@ -88,7 +88,7 @@ Map map_value(const View& v) { if (NLOHMANN_VIEW_UNLIKELY(!v.is_object())) { - throw_type_error(exception_id::type_mismatch, "type must be object, but is ", v.type_name()); + throw_type_error(site_of(v), exception_id::type_mismatch, "type must be object, but is ", v.type_name()); } Map r; for (auto it = v.begin(); it != v.end(); ++it) diff --git a/include/nlohmann/json_view.hpp b/include/nlohmann/json_view.hpp index 097e489a4..043d09036 100644 --- a/include/nlohmann/json_view.hpp +++ b/include/nlohmann/json_view.hpp @@ -285,7 +285,7 @@ class basic_json_view { return {}; } - detail::view::throw_type_error(detail::exception_id::subscript_wrong_type, "cannot use operator[] with a string argument with ", type_name()); + detail::view::throw_type_error(site(), detail::exception_id::subscript_wrong_type, "cannot use operator[] with a string argument with ", type_name()); } return lookup(key); } @@ -311,7 +311,7 @@ class basic_json_view { return {}; } - detail::view::throw_type_error(detail::exception_id::subscript_wrong_type, "cannot use operator[] with a numeric argument with ", type_name()); + detail::view::throw_type_error(site(), detail::exception_id::subscript_wrong_type, "cannot use operator[] with a numeric argument with ", type_name()); } return idx < m_node->len ? element(idx) : basic_json_view(); } @@ -344,12 +344,12 @@ class basic_json_view { if (NLOHMANN_VIEW_UNLIKELY(!is_object())) { - detail::view::throw_type_error(detail::exception_id::at_wrong_type, "cannot use at() with ", type_name()); + detail::view::throw_type_error(site(), detail::exception_id::at_wrong_type, "cannot use at() with ", type_name()); } const basic_json_view r = lookup(key); if (NLOHMANN_VIEW_UNLIKELY(r.is_discarded())) { - detail::view::throw_key_not_found(std::string(key.data(), key.size())); + detail::view::throw_key_not_found(site(), std::string(key.data(), key.size())); } return r; } @@ -370,11 +370,11 @@ class basic_json_view { if (NLOHMANN_VIEW_UNLIKELY(!is_array())) { - detail::view::throw_type_error(detail::exception_id::at_wrong_type, "cannot use at() with ", type_name()); + detail::view::throw_type_error(site(), detail::exception_id::at_wrong_type, "cannot use at() with ", type_name()); } if (NLOHMANN_VIEW_UNLIKELY(idx >= m_node->len)) { - detail::view::throw_array_index_out_of_range(idx); + detail::view::throw_array_index_out_of_range(site(), idx); } return element(idx); } @@ -402,7 +402,7 @@ class basic_json_view { if (NLOHMANN_VIEW_UNLIKELY(!is_object())) { - detail::view::throw_type_error(detail::exception_id::value_wrong_type, "cannot use value() with ", type_name()); + detail::view::throw_type_error(site(), detail::exception_id::value_wrong_type, "cannot use value() with ", type_name()); } const basic_json_view r = lookup(key); return r.is_discarded() ? default_value : r.template get(); @@ -412,7 +412,7 @@ class basic_json_view { if (NLOHMANN_VIEW_UNLIKELY(!is_object())) { - detail::view::throw_type_error(detail::exception_id::value_wrong_type, "cannot use value() with ", type_name()); + detail::view::throw_type_error(site(), detail::exception_id::value_wrong_type, "cannot use value() with ", type_name()); } // (the default is made only if there is no such member) const basic_json_view r = lookup(key); @@ -427,7 +427,7 @@ class basic_json_view { if (NLOHMANN_VIEW_UNLIKELY(!is_structured())) { - detail::view::throw_type_error(detail::exception_id::value_wrong_type, "cannot use value() with ", type_name()); + detail::view::throw_type_error(site(), detail::exception_id::value_wrong_type, "cannot use value() with ", type_name()); } const basic_json_view r = detail::view::resolve_pointer(*this, detail::json_pointer_access::reference_tokens(ptr), detail::view::pointer_mode::value); return r.is_discarded() ? default_value : r.template get(); @@ -437,7 +437,7 @@ class basic_json_view { if (NLOHMANN_VIEW_UNLIKELY(!is_structured())) { - detail::view::throw_type_error(detail::exception_id::value_wrong_type, "cannot use value() with ", type_name()); + detail::view::throw_type_error(site(), detail::exception_id::value_wrong_type, "cannot use value() with ", type_name()); } const basic_json_view r = detail::view::resolve_pointer(*this, detail::json_pointer_access::reference_tokens(ptr), detail::view::pointer_mode::value); return r.is_discarded() ? string_t(default_value) : r.template get(); @@ -450,7 +450,7 @@ class basic_json_view const iterator it = begin(); if (NLOHMANN_VIEW_UNLIKELY(it == end())) { - detail::view::throw_invalid_iterator(detail::exception_id::iterator_value_unavailable, "cannot get value"); + detail::view::throw_invalid_iterator(site(), detail::exception_id::iterator_value_unavailable, "cannot get value"); } return *it; } @@ -606,7 +606,7 @@ class basic_json_view { if (NLOHMANN_VIEW_UNLIKELY(!is_string())) { - detail::view::throw_type_error(detail::exception_id::type_mismatch, "type must be string, but is ", type_name()); + detail::view::throw_type_error(site(), detail::exception_id::type_mismatch, "type must be string, but is ", type_name()); } return {m_doc->str(*m_node), m_node->len}; } @@ -618,7 +618,7 @@ class basic_json_view { if (NLOHMANN_VIEW_UNLIKELY(!is_number())) { - detail::view::throw_type_error(detail::exception_id::type_mismatch, "type must be number, but is ", type_name()); + detail::view::throw_type_error(site(), detail::exception_id::type_mismatch, "type must be number, but is ", type_name()); } return {m_doc->str(*m_node), detail::view::number_length(*m_node)}; } @@ -764,12 +764,19 @@ class basic_json_view template friend class basic_json_view; template friend class detail::view::editor; template friend class detail::view::view_side; + friend struct detail::view::view_access; friend iterator; basic_json_view(const document_data* d, const node* n) noexcept : m_doc(d), m_node(n) {} + /// this value as the context of an exception + detail::view::error_site site() const noexcept + { + return {m_doc, m_node}; + } + using json_side_t = detail::view::json_side; static detail::view::view_side side(const basic_json_view& v) noexcept @@ -876,7 +883,7 @@ class basic_json_view { if (NLOHMANN_VIEW_UNLIKELY(!is_boolean())) { - detail::view::throw_type_error(detail::exception_id::type_mismatch, "type must be boolean, but is ", type_name()); + detail::view::throw_type_error(site(), detail::exception_id::type_mismatch, "type must be boolean, but is ", type_name()); } return (m_node->flags & detail::view::node_flags::is_true) != 0; } @@ -886,7 +893,7 @@ class basic_json_view { if (NLOHMANN_VIEW_UNLIKELY(m_node == nullptr)) { - detail::view::throw_type_error(detail::exception_id::type_mismatch, "type must be number, but is ", type_name()); + detail::view::throw_type_error(site(), detail::exception_id::type_mismatch, "type must be number, but is ", type_name()); } return detail::view::arithmetic_value(*m_doc, *m_node); } @@ -895,7 +902,7 @@ class basic_json_view { if (NLOHMANN_VIEW_UNLIKELY(!is_null())) { - detail::view::throw_type_error(detail::exception_id::type_mismatch, "type must be null, but is ", type_name()); + detail::view::throw_type_error(site(), detail::exception_id::type_mismatch, "type must be null, but is ", type_name()); } return nullptr; } @@ -1310,7 +1317,7 @@ class basic_json_document template::value, int>::type = 0> view_type set(view_type array, I idx, V && value) { - return editor().set(array, index(idx), std::forward(value)); + return editor().set(array, index(array, idx), std::forward(value)); } /// set the value at a JSON pointer: its parent must exist; an object @@ -1347,7 +1354,7 @@ class basic_json_document const std::size_t idx = pointer_index(token); if (idx >= 0xFFFFFFFFu) { - detail::view::throw_array_index_out_of_range(idx); + detail::view::throw_array_index_out_of_range(detail::view::site_of(parent), idx); } for (std::size_t i = 0; i < idx; ++i) { @@ -1381,7 +1388,7 @@ class basic_json_document template::value, int>::type = 0> view_type insert(view_type array, I idx, V && value) { - return editor().insert(array, index(idx), std::forward(value)); + return editor().insert(array, index(array, idx), std::forward(value)); } /// remove all members with this key; returns their number @@ -1394,7 +1401,7 @@ class basic_json_document template::value, int>::type = 0> void erase(view_type array, I idx) { - editor().erase(array, index(idx)); + editor().erase(array, index(array, idx)); } /// remove the value at a JSON pointer; returns the number of removed @@ -1428,25 +1435,25 @@ class basic_json_document return detail::view::editor(*m_data); } - /// an index (out_of_range.401 if negative) + /// an index into @a array (out_of_range.401 if negative) template - static std::size_t index(I idx) + static std::size_t index(const view_type& array, I idx) { - return index(idx, std::is_signed {}); + return index(array, idx, std::is_signed {}); } template - static std::size_t index(I idx, std::true_type /*signed*/) + static std::size_t index(const view_type& array, I idx, std::true_type /*signed*/) { if (idx < 0) { - detail::view::throw_array_index_out_of_range(idx); + detail::view::throw_array_index_out_of_range(detail::view::site_of(array), idx); } return static_cast(idx); } template - static std::size_t index(I idx, std::false_type /*unsigned*/) + static std::size_t index(const view_type& /*array*/, I idx, std::false_type /*unsigned*/) { return static_cast(idx); } diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 7b54e0a89..1fc214c5f 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -2720,7 +2720,7 @@ JSON_HEDLEY_DIAGNOSTIC_POP return ej_pair.second == j; \ }); \ if (it != std::end(m)) e = it->first; \ - else ::nlohmann::detail::templated_json_throw(nlohmann::detail::out_of_range::create(nlohmann::detail::exception_id::value_out_of_range, nlohmann::detail::concat("enum value out of range for " #ENUM_TYPE ": ", nlohmann::detail::from_json_dump(j)), nlohmann::detail::exception_context(j))); \ + else ::nlohmann::detail::templated_json_throw(nlohmann::detail::out_of_range::create(nlohmann::detail::exception_id::value_out_of_range, nlohmann::detail::concat(nlohmann::detail::exception_prefix(j), "enum value out of range for " #ENUM_TYPE ": ", nlohmann::detail::from_json_dump(j)), nlohmann::detail::exception_context(j))); \ } // Ugly macros to avoid uglier copy-paste when specializing basic_json. They @@ -5928,6 +5928,28 @@ std::nullptr_t exception_context(const View& /*unused*/) noexcept return nullptr; } +namespace view +{ +// the path and bytes of the value of a view (defined in ) +template +std::string diagnostics_of(const View& v); +} // namespace view + +// what goes in front of the message of an exception about j: nothing for a +// basic_json value (exception_context() passes the value itself), the context +// basic_json would add (JSON_DIAGNOSTICS, JSON_DIAGNOSTIC_POSITIONS) for a view +template::value, int> = 0> +const char* exception_prefix(const BasicJsonType& /*unused*/) noexcept +{ + return ""; +} + +template::value, int> = 0> +std::string exception_prefix(const View& v) +{ + return view::diagnostics_of(v); +} + /*! @brief throws because @a j does not have the type a conversion expects @param[in] expected the expected type(s), e.g. "array" or "binary or array" @@ -5939,7 +5961,7 @@ JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_type_must_be(const cha { static_cast(expected); // unused when JSON_NOEXCEPTION is defined static_cast(j); - JSON_THROW(type_error::create(exception_id::type_mismatch, concat("type must be ", expected, ", but is ", j.type_name()), exception_context(j))); + JSON_THROW(type_error::create(exception_id::type_mismatch, concat(exception_prefix(j), "type must be ", expected, ", but is ", j.type_name()), exception_context(j))); } /*! diff --git a/single_include/nlohmann/json_view.hpp b/single_include/nlohmann/json_view.hpp index 408d845d4..e24f125bf 100644 --- a/single_include/nlohmann/json_view.hpp +++ b/single_include/nlohmann/json_view.hpp @@ -4395,13 +4395,44 @@ NLOHMANN_JSON_NAMESPACE_END #include // min #include // size_t -#include // string +#include // string, to_string +#include // vector // #include // #include +// #include + // #include +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // array +#include // size_t +#include // int64_t, uint8_t +#include // string +#include // vector + +// #include +// #include + +// #include + +// #include + +// #include + +// #include + NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail @@ -4409,8 +4440,721 @@ namespace detail namespace view { +#if JSON_DIAGNOSTIC_POSITIONS +/*! +@brief the source positions of the values materialize() replays + +Stands in for the lexer of json_sax_dom_parser, which reads the positions of +the values it stores from these three functions (diagnostic_positions). npos +is a value without a source position. One position_source serves a whole +materialize() call: before each value is handed to the handler, the range of +that value is set (or cleared), overwriting that of the value before. +*/ +struct position_source +{ + /// the token, of which diagnostic_positions only reads the length + struct token + { + std::size_t length; + std::size_t size() const noexcept + { + return length; + } + }; + + std::size_t start = std::string::npos; ///< the first byte of the next value + std::size_t end = std::string::npos; ///< one past the next value (start_object/start_array: one past the bracket) + + std::size_t get_chars_read_total() const noexcept + { + return end; + } + std::size_t get_token_start_position() const noexcept + { + return start; + } + token get_string() const noexcept + { + return token{end - start}; + } + + /// the range [first, last), or none if either is npos + void set(std::size_t first, std::size_t last) noexcept + { + if (first == std::string::npos || last == std::string::npos) + { + clear(); + return; + } + start = first; + end = last; + } + + /// no range: the next value has no source position + void clear() noexcept + { + start = std::string::npos; + end = std::string::npos; + } + + /// one past the closing quote of the string whose opening quote is at + /// src[quote], or npos (the scan is bounded by size, as images can be + /// loaded without a check) + static std::size_t string_end(const char* src, std::size_t size, std::size_t quote) noexcept + { + for (std::size_t i = quote + 1; i < size; ++i) + { + if (src[i] == '\\') + { + ++i; + } + else if (src[i] == '"') + { + return i + 1; + } + } + return std::string::npos; + } + + /// one past the closing bracket of the array or object whose last element + /// ends at src[pos] (or whose opening bracket is at src[pos - 1]), or npos; + /// in between, there can only be whitespace, comments, and a trailing comma + /// (the source was validated when it was parsed) + static std::size_t container_end(const char* src, std::size_t size, std::size_t pos) noexcept + { + const auto* const b = reinterpret_cast(src); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the text as bytes, as the builder reads it + const unsigned char* const e = b + size; + while (pos < size) + { + switch (src[pos]) + { + case ' ': + case '\t': + case '\n': + case '\r': + case ',': + ++pos; + break; + case ']': + case '}': + return pos + 1; + case '/': + if (pos + 1 < size && src[pos + 1] == '/') + { + pos = static_cast(skip_line_comment(b + pos + 2, e) - b); + break; + } + if (pos + 1 < size && src[pos + 1] == '*') + { + // (unterminated: pos is size, which ends the loop) + pos = static_cast(find_block_comment_end(b + pos + 2, e) - b) + 2; + break; + } + return std::string::npos; + default: + return std::string::npos; + } + } + return std::string::npos; + } + + /// the range of the scalar n of document d + void set_scalar(const document_data& d, const node& n) noexcept + { + if ((n.flags & node_flags::is_new) != 0) + { + clear(); + return; + } + std::size_t first = std::string::npos; + std::size_t last = std::string::npos; + switch (n.kind) + { + case node_kind::string: + if ((n.flags & node_flags::storage) == 0) + { + // off is the first byte after the opening quote + if (n.off != 0) + { + first = n.off - 1; + last = n.off + std::size_t{n.len} + 1; + } + } + else if ((n.flags & node_flags::storage) == node_flags::escaped && n.next != 0) + { + // next is the offset of the opening quote plus one + first = n.next - 1; + last = string_end(d.base[0], d.size, first); + } + break; + case node_kind::number_integer: + case node_kind::number_unsigned: + case node_kind::number_float: + if ((n.flags & node_flags::storage) == 0) + { + first = n.off; + last = n.off + number_length(n); + } + break; + case node_kind::boolean: + first = n.off; + last = n.off + ((n.flags & node_flags::is_true) != 0 ? 4u : 5u); + break; + case node_kind::null: + first = n.off; + last = n.off + 4u; + break; + case node_kind::object: + case node_kind::array: + case node_kind::link: + default: + break; + } + // (a crafted image can point past the text) + if (last == std::string::npos || last > d.size) + { + clear(); + } + else + { + set(first, last); + } + } +}; +#endif + +/*! +@brief the basic_json value of the subtree at n + +The subtree is replayed into the SAX handler that parse() uses to build its +values, so the result is the value parse() would produce: duplicate keys keep +the last value, and with JSON_DIAGNOSTICS the parent pointers are set. It is +iterative, so the nesting depth is limited by memory only, as for parse(). + +With JSON_DIAGNOSTIC_POSITIONS, the values get their source positions as with +parse(): position_source stands in for the lexer. Values written by edits have +none (npos), and neither has an array or object whose elements changed, or +whose last element has none (the end of its closing bracket is found by +scanning from the end of the last element). +*/ +template +BasicJsonType materialize(const document_data& d, const node* n) +{ + using string_t = typename BasicJsonType::string_t; + using input_t = iterator_input_adapter; +#if JSON_DIAGNOSTIC_POSITIONS + using lexer_t = position_source; +#else + using lexer_t = lexer; // (the default: unused, as no lexer is passed) +#endif + using sax_t = json_sax_dom_parser; + using nav = navigation; + + // the open containers: their next element (or key of the next member), + // their end, and whether they are objects; for the first 16 levels on the + // stack, else sized once from the document's deepest nesting (for a + // subtree an upper bound) + struct frame + { + const node* pos; + const node* end; + bool object; +#if JSON_DIAGNOSTIC_POSITIONS + std::size_t start; ///< the opening bracket (npos: unknown) + std::size_t last_end; ///< the end of the last element read (npos: unknown) +#endif + }; + +#if !JSON_DIAGNOSTIC_POSITIONS + // a scalar needs neither the handler nor the stack of open containers + // (with positions, it goes through the handler, which sets them) + switch (static_cast(n->kind)) + { + case value_t::string: + return BasicJsonType(string_t(d.str(*n), n->len)); + case value_t::number_integer: + return BasicJsonType(static_cast(static_cast(integer_bits(*n)))); + case value_t::number_unsigned: + return BasicJsonType(static_cast(integer_bits(*n))); + case value_t::number_float: + return BasicJsonType(float_value(d, *n)); + case value_t::boolean: + return BasicJsonType((n->flags & node_flags::is_true) != 0); + case value_t::null: + return BasicJsonType(nullptr); + case value_t::object: + case value_t::array: + case value_t::binary: + case value_t::discarded: + default: + break; + } +#endif + + BasicJsonType result; +#if JSON_DIAGNOSTIC_POSITIONS + position_source positions; + sax_t sax(result, true, &positions); +#else + sax_t sax(result, true); +#endif + const string_t no_token {}; + // (shallow subtrees on the stack: a call for a small value then allocates only for the value) + std::array shallow; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): filled as containers open + std::vector deep{}; + frame* open = shallow.data(); + std::size_t capacity = shallow.size(); + if (d.max_depth > capacity) + { + deep.resize(d.max_depth); + open = deep.data(); + capacity = deep.size(); + } + std::size_t depth = 0; + for (;;) + { +#if JSON_DIAGNOSTIC_POSITIONS + if (!is_container(*n)) + { + positions.set_scalar(d, *n); + if (depth != 0) + { + open[depth - 1].last_end = positions.end; + } + } +#endif + // false positive: n comes from nav::value(), which never returns null for a valid index + // @infer-ignore NULLPTR_DEREFERENCE + switch (static_cast(n->kind)) + { + case value_t::object: + case value_t::array: + { + const bool object = n->kind == node_kind::object; +#if JSON_DIAGNOSTIC_POSITIONS + // (moved containers are also new) + if ((n->flags & node_flags::is_new) != 0 || n->off >= d.size) + { + positions.clear(); + } + else + { + positions.set(std::size_t{n->off}, std::size_t{n->off} + 1); + } +#endif + if (object) + { + sax.start_object(n->len); + } + else + { + sax.start_array(n->len); + } + if (NLOHMANN_VIEW_UNLIKELY(depth == capacity)) + { + // (deeper than the recorded nesting, e.g. after edits) + deep.assign(open, open + depth); + deep.resize(depth * 2); + open = deep.data(); + capacity = deep.size(); + } + // (with positions: until an element is read, the container's + // own bracket is the last thing read) + open[depth++] = frame{nav::first(d, n), nav::end(d, n), object +#if JSON_DIAGNOSTIC_POSITIONS + , positions.start, positions.end +#endif + }; + break; + } + case value_t::string: + { + string_t s(d.str(*n), n->len); + sax.string(s); + break; + } + case value_t::number_integer: + sax.number_integer(static_cast(static_cast(integer_bits(*n)))); + break; + case value_t::number_unsigned: + sax.number_unsigned(static_cast(integer_bits(*n))); + break; + case value_t::number_float: + sax.number_float(float_value(d, *n), no_token); + break; + case value_t::boolean: + sax.boolean((n->flags & node_flags::is_true) != 0); + break; + case value_t::null: + case value_t::binary: + case value_t::discarded: + default: + // false positive: the parser's frame stack points into its own storage (never null) + // @infer-ignore NULLPTR_DEREFERENCE + sax.null(); + break; + } + + // the next value: close finished containers, then read the key + for (;;) + { + if (depth == 0) + { + return result; + } + frame& f = open[depth - 1]; + if (f.pos == f.end) + { +#if JSON_DIAGNOSTIC_POSITIONS + if (f.start == std::string::npos || f.last_end == std::string::npos) + { + positions.clear(); + } + else + { + const std::size_t end = position_source::container_end(d.base[0], d.size, f.last_end); + if (end == std::string::npos) + { + positions.clear(); + } + else + { + // the closing bracket, as the lexer reports it (end_object() + // and end_array() only read its end; the start of the + // container was set when it was opened) + positions.set(end - 1, end); + } + } + if (depth > 1) + { + open[depth - 2].last_end = positions.end; + } +#endif + if (f.object) + { + sax.end_object(); + } + else + { + sax.end_array(); + } + --depth; + continue; + } + const node* entry = f.pos; + if (f.object) + { + string_t key(d.str(*entry), entry->len); + sax.key(key); + ++entry; + } + n = nav::value(entry); + f.pos = document_data::after(entry); + break; + } + } +} + +} // namespace view +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + +// #include + + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ +namespace view +{ + +/*! +@brief the value an exception is about + +basic_json passes the value to the exception, which then names its path +(JSON_DIAGNOSTICS) and its bytes in the source (JSON_DIAGNOSTIC_POSITIONS). A +view's value has no parent pointer and no stored positions: both are found +from the document and the node, when the exception is thrown (so only the +error path pays for them). Without the two macros, a site is empty. +*/ +class error_site +{ + public: + error_site() noexcept = default; + +#if JSON_DIAGNOSTICS || JSON_DIAGNOSTIC_POSITIONS + error_site(const document_data* d, const node* n) noexcept + : m_doc(d), m_node(n) + {} + + const document_data* doc() const noexcept + { + return m_doc; + } + const node* value() const noexcept + { + return m_node; + } + + private: + const document_data* m_doc = nullptr; + const node* m_node = nullptr; +#else + error_site(const document_data* /*d*/, const node* /*n*/) noexcept {} +#endif +}; + +/// the site of a view's value (the views make this a friend) +struct view_access +{ + template + static error_site site(const View& v) noexcept + { + return {v.m_doc, v.m_node}; + } +}; + +template +error_site site_of(const View& v) noexcept +{ + static_assert(is_json_view::value, "the site of a value is only defined for views"); + return view_access::site(v); +} + +#if JSON_DIAGNOSTICS +/*! +@brief the JSON pointer tokens from the root of document @a d to node @a target + +Nodes have no parent index, so the path is found by descending from the root: +before the first edit, only into the subtree that contains the target (a +container spans the next `next` nodes); after edits, elements can live in +other sequences, so every container is searched. Iterative (the nesting depth +is limited by memory only). False if the target is not found (then tokens is +unspecified). +*/ +template +bool value_path(const document_data& d, const node* target, std::vector& tokens) +{ + using nav = navigation; + struct frame + { + const node* pos; + const node* end; + std::size_t index; + bool object; + }; + const node* const root = d.nodes; + if (target == root) + { + return true; + } + if (!is_container(*root)) + { + return false; + } + std::vector stack; + stack.push_back(frame{nav::first(d, root), nav::end(d, root), 0, root->kind == node_kind::object}); + // (tokens[i] leads from stack[i] to stack[i + 1], or to the target) + for (;;) + { + // a copy: pop_back() and push_back() below change the stack + const frame f = stack.back(); + if (f.pos == f.end) + { + stack.pop_back(); + if (stack.empty()) + { + return false; + } + tokens.pop_back(); + continue; + } + const node* entry = f.pos; + std::string token; + if (f.object) + { + // (as basic_json: the key up to a null byte) + token.assign(d.str(*entry), entry->len); + const std::size_t nul = token.find('\0'); + if (nul != std::string::npos) + { + token.resize(nul); + } + ++entry; + } + else + { + token = std::to_string(f.index); + } + const node* const child = nav::value(entry); + if (child == target) + { + tokens.push_back(token); + return true; + } + stack.back().pos = document_data::after(entry); + ++stack.back().index; + if (is_container(*child) && (nav::editable() || (target > child && target < child + child->next))) + { + tokens.push_back(token); + stack.push_back(frame{nav::first(d, child), nav::end(d, child), 0, child->kind == node_kind::object}); + } + } +} +#endif + +#if JSON_DIAGNOSTIC_POSITIONS +/*! +@brief the bytes [start, end) of node @a n in the source, as materialize() sets them + +Scalars as position_source::set_scalar; an array or object from its opening +bracket to its closing bracket, which is found by following the chain of last +elements to a scalar (or an empty container) and scanning from there to the +closing brackets. None (npos) for values written by edits, and for containers +whose last element has none, as with materialize(). +*/ +template +void value_range(const document_data& d, const node* n, std::size_t& start, std::size_t& end) +{ + using nav = navigation; + position_source p; + start = std::string::npos; + end = std::string::npos; + if (!is_container(*n)) + { + p.set_scalar(d, *n); + start = p.start; + end = p.end; + return; + } + if ((n->flags & node_flags::is_new) != 0 || n->off >= d.size) + { + return; + } + // the open brackets: n, then the last element as long as it is a non-empty container + std::size_t closing = 0; + std::size_t pos = std::string::npos; + for (const node* c = n;;) + { + if (!is_container(*c)) + { + p.set_scalar(d, *c); + pos = p.end; + break; + } + if ((c->flags & node_flags::is_new) != 0 || c->off >= d.size) + { + return; + } + ++closing; + const node* const first = nav::first(d, c); + const node* const last_end = nav::end(d, c); + if (first == last_end) + { + pos = std::size_t{c->off} + 1; + break; + } + const std::size_t value_offset = c->kind == node_kind::object ? 1 : 0; + const node* last = first; + for (const node* e = document_data::after(first + value_offset); e != last_end; e = document_data::after(e + value_offset)) + { + last = e; + } + c = nav::value(last + value_offset); + } + for (; closing != 0 && pos != std::string::npos; --closing) + { + pos = position_source::container_end(d.base[0], d.size, pos); + } + if (pos != std::string::npos) + { + start = n->off; + end = pos; + } +} +#endif + +/// the context basic_json puts in front of the message of an exception about +/// a value: "(/path) " with JSON_DIAGNOSTICS (none for the root), then +/// "(bytes a-b) " with JSON_DIAGNOSTIC_POSITIONS (with JSON_DIAGNOSTICS, only +/// if there is a path) +/// +/// This and the throw helpers that take an error_site are templates, so that +/// the path and range code is instantiated only where an exception can be +/// thrown: instantiated in every translation unit (as a non-template inline +/// function makes it), it changed GCC's early inlining of unrelated view +/// functions, so that GCC 13 no longer inlined a recursive walk over a view +/// into itself (2.7 times the instructions). +#if JSON_DIAGNOSTICS || JSON_DIAGNOSTIC_POSITIONS +template +std::string diagnostics(const error_site site) +{ + if (site.doc() == nullptr || site.value() == nullptr) + { + return ""; + } + const document_data& d = *site.doc(); + const bool edited = d.edits != nullptr; + std::string context; +#if JSON_DIAGNOSTICS + std::vector tokens; + const bool found = edited ? value_path(d, site.value(), tokens) : value_path(d, site.value(), tokens); + if (!found || tokens.empty()) + { + // context is still empty; returning it keeps the copy elided (-Wnrvo) + return context; + } + std::string path; + for (const std::string& token : tokens) + { + path.push_back('/'); + path += detail::escape(token); + } + context = concat('(', path, ") "); +#endif +#if JSON_DIAGNOSTIC_POSITIONS + std::size_t start = 0; + std::size_t end = 0; + if (edited) + { + value_range(d, site.value(), start, end); + } + else + { + value_range(d, site.value(), start, end); + } + if (start != std::string::npos && end != std::string::npos) + { + context += concat("(bytes ", std::to_string(start), "-", std::to_string(end), ") "); + } +#endif + return context; +} +#else +template +std::string diagnostics(const error_site /*site*/) +{ + return ""; +} +#endif + +/// the context of an exception about view @a v (see exception_prefix()) +template +std::string diagnostics_of(const View& v) +{ + return diagnostics<>(site_of(v)); +} + // Exceptions are thrown out of line, so that the accessors that may throw stay -// small enough to be inlined. +// small enough to be inlined. The overloads with an error_site put its context +// in front of the message. +// +// The overloads without one are for exceptions that are not about a value of a +// document: limits and invalid input of the editing and image functions, a +// discarded view (it has no node), a view of another document (its path would +// be one in that document), and those that basic_json creates without a +// context too (the errors of json_pointer, invalid UTF-8). The one exception +// about a value that has no site is key(), see view_iterator::key(). [[noreturn]] NLOHMANN_VIEW_NOINLINE inline void throw_type_error(exception_id id, const char* prefix, const char* type) { @@ -4432,6 +5176,24 @@ namespace view NLOHMANN_VIEW_THROW(invalid_iterator::create(id, msg, nullptr)); } +template +[[noreturn]] NLOHMANN_VIEW_NOINLINE void throw_type_error(const error_site site, exception_id id, const char* prefix, const char* type) +{ + NLOHMANN_VIEW_THROW(type_error::create(id, concat(diagnostics<>(site), prefix, type), nullptr)); +} + +template +[[noreturn]] NLOHMANN_VIEW_NOINLINE void throw_out_of_range(const error_site site, exception_id id, const std::string& msg) +{ + NLOHMANN_VIEW_THROW(out_of_range::create(id, concat(diagnostics<>(site), msg), nullptr)); +} + +template +[[noreturn]] NLOHMANN_VIEW_NOINLINE void throw_invalid_iterator(const error_site site, exception_id id, const char* msg) +{ + NLOHMANN_VIEW_THROW(invalid_iterator::create(id, concat(diagnostics<>(site), msg), nullptr)); +} + /// a parse error without a position (as those of json_pointer) [[noreturn]] NLOHMANN_VIEW_NOINLINE inline void throw_parse_error(exception_id id, const std::string& msg) { @@ -4440,22 +5202,23 @@ namespace view /// out_of_range.401: the array index @a idx is not below the size template -[[noreturn]] NLOHMANN_VIEW_NOINLINE void throw_array_index_out_of_range(const Index idx) +[[noreturn]] NLOHMANN_VIEW_NOINLINE void throw_array_index_out_of_range(const error_site site, const Index idx) { - throw_out_of_range(exception_id::array_index_out_of_range, concat("array index ", std::to_string(idx), " is out of range")); + throw_out_of_range(site, exception_id::array_index_out_of_range, concat("array index ", std::to_string(idx), " is out of range")); } /// out_of_range.402: the JSON Pointer token '-' of an array of @a size elements -[[noreturn]] NLOHMANN_VIEW_NOINLINE inline void throw_array_index_past_end(std::size_t size) +template +[[noreturn]] NLOHMANN_VIEW_NOINLINE void throw_array_index_past_end(const error_site site, std::size_t size) { - throw_out_of_range(exception_id::pointer_past_the_end_index, concat("array index '-' (", std::to_string(size), ") is out of range")); + throw_out_of_range(site, exception_id::pointer_past_the_end_index, concat("array index '-' (", std::to_string(size), ") is out of range")); } /// out_of_range.403: the object has no member @a key template -[[noreturn]] NLOHMANN_VIEW_NOINLINE void throw_key_not_found(const Key& key) +[[noreturn]] NLOHMANN_VIEW_NOINLINE void throw_key_not_found(const error_site site, const Key& key) { - throw_out_of_range(exception_id::key_not_found, concat("key '", key, "' not found")); + throw_out_of_range(site, exception_id::key_not_found, concat("key '", key, "' not found")); } /// SAX handler of the error replay: accepts every event without storing it, @@ -5492,7 +6255,7 @@ class editor node* const o = own(object); if (o->kind != node_kind::object && o->kind != node_kind::null) { - throw_type_error(exception_id::subscript_wrong_type, "cannot use operator[] with a string argument with ", object.type_name()); + throw_type_error(site_of(object), exception_id::subscript_wrong_type, "cannot use operator[] with a string argument with ", object.type_name()); } check_utf8(key.data(), key.size()); const encoded e = encode(std::forward(value)); @@ -5543,9 +6306,9 @@ class editor node* const a = own(array); if (a->kind != node_kind::array) { - throw_type_error(exception_id::subscript_wrong_type, "cannot use operator[] with a numeric argument with ", array.type_name()); + throw_type_error(site_of(array), exception_id::subscript_wrong_type, "cannot use operator[] with a numeric argument with ", array.type_name()); } - check_index(idx, a->len); + check_index(a, idx, a->len); const encoded e = encode(std::forward(value)); node* const slot = const_cast(nav::value(element_at(m_doc, a, idx))); // NOLINT(cppcoreguidelines-pro-type-const-cast) assign(slot, e, a, true); @@ -5560,7 +6323,7 @@ class editor node* const a = own(array); if (a->kind != node_kind::array && a->kind != node_kind::null) { - throw_type_error(exception_id::push_back_wrong_type, "cannot use push_back() with ", array.type_name()); + throw_type_error(site_of(array), exception_id::push_back_wrong_type, "cannot use push_back() with ", array.type_name()); } const encoded e = encode(std::forward(value)); if (a->kind == node_kind::null) @@ -5584,9 +6347,9 @@ class editor node* const a = own(array); if (a->kind != node_kind::array) { - throw_type_error(exception_id::insert_wrong_type, "cannot use insert() with ", array.type_name()); + throw_type_error(site_of(array), exception_id::insert_wrong_type, "cannot use insert() with ", array.type_name()); } - check_index(idx, a->len + 1); + check_index(a, idx, a->len + 1); const encoded e = encode(std::forward(value)); node* const slot = new_slot(e); node* const h = block_of(m_doc, a, 1); @@ -5604,7 +6367,7 @@ class editor node* const o = own(object); if (o->kind != node_kind::object) { - throw_type_error(exception_id::erase_wrong_type, "cannot use erase() with ", object.type_name()); + throw_type_error(site_of(object), exception_id::erase_wrong_type, "cannot use erase() with ", object.type_name()); } for (const node* k = nav::first(m_doc, o), *end = nav::end(m_doc, o); k != end; k = document_data::after(k + 1)) { @@ -5622,9 +6385,9 @@ class editor node* const a = own(array); if (a->kind != node_kind::array) { - throw_type_error(exception_id::erase_wrong_type, "cannot use erase() with ", array.type_name()); + throw_type_error(site_of(array), exception_id::erase_wrong_type, "cannot use erase() with ", array.type_name()); } - check_index(idx, a->len); + check_index(a, idx, a->len); node* const h = block_of(m_doc, a, 0); std::memmove(h + 1 + idx, h + 2 + idx, (h->next - 2 - idx) * sizeof(node)); --h->next; @@ -5651,11 +6414,11 @@ class editor return const_cast(v.m_node); // NOLINT(cppcoreguidelines-pro-type-const-cast): the nodes belong to this document } - static void check_index(std::size_t idx, std::size_t limit) + void check_index(const node* array, std::size_t idx, std::size_t limit) const { if (idx >= limit) { - throw_array_index_out_of_range(idx); + throw_array_index_out_of_range(error_site(&m_doc, array), idx); } } @@ -7379,7 +8142,10 @@ class view_iterator } /// the key of the current object member; throws invalid_iterator.207 for - /// other iterators, like basic_json's iterators + /// other iterators, like basic_json's iterators (but without the context + /// JSON_DIAGNOSTICS and JSON_DIAGNOSTIC_POSITIONS add: the iterator does not + /// know the value it iterates over, and keeping it would make iterators + /// larger, which keeps GCC from inlining recursive walks) string_view_t key() const { if (NLOHMANN_VIEW_UNLIKELY(m_value_offset == 0)) @@ -7543,458 +8309,6 @@ NLOHMANN_JSON_NAMESPACE_END // #include // #include -// __ _____ _____ _____ -// __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.12.0 -// |_____|_____|_____|_|___| https://github.com/nlohmann/json -// -// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann -// SPDX-License-Identifier: MIT - - - -#include // array -#include // size_t -#include // int64_t, uint8_t -#include // string -#include // vector - -// #include -// #include - -// #include - -// #include - -// #include - -// #include - - -NLOHMANN_JSON_NAMESPACE_BEGIN -namespace detail -{ -namespace view -{ - -#if JSON_DIAGNOSTIC_POSITIONS -/*! -@brief the source positions of the values materialize() replays - -Stands in for the lexer of json_sax_dom_parser, which reads the positions of -the values it stores from these three functions (diagnostic_positions). npos -is a value without a source position. One position_source serves a whole -materialize() call: before each value is handed to the handler, the range of -that value is set (or cleared), overwriting that of the value before. -*/ -struct position_source -{ - /// the token, of which diagnostic_positions only reads the length - struct token - { - std::size_t length; - std::size_t size() const noexcept - { - return length; - } - }; - - std::size_t start = std::string::npos; ///< the first byte of the next value - std::size_t end = std::string::npos; ///< one past the next value (start_object/start_array: one past the bracket) - - std::size_t get_chars_read_total() const noexcept - { - return end; - } - std::size_t get_token_start_position() const noexcept - { - return start; - } - token get_string() const noexcept - { - return token{end - start}; - } - - /// the range [first, last), or none if either is npos - void set(std::size_t first, std::size_t last) noexcept - { - if (first == std::string::npos || last == std::string::npos) - { - clear(); - return; - } - start = first; - end = last; - } - - /// no range: the next value has no source position - void clear() noexcept - { - start = std::string::npos; - end = std::string::npos; - } - - /// one past the closing quote of the string whose opening quote is at - /// src[quote], or npos (the scan is bounded by size, as images can be - /// loaded without a check) - static std::size_t string_end(const char* src, std::size_t size, std::size_t quote) noexcept - { - for (std::size_t i = quote + 1; i < size; ++i) - { - if (src[i] == '\\') - { - ++i; - } - else if (src[i] == '"') - { - return i + 1; - } - } - return std::string::npos; - } - - /// one past the closing bracket of the array or object whose last element - /// ends at src[pos] (or whose opening bracket is at src[pos - 1]), or npos; - /// in between, there can only be whitespace, comments, and a trailing comma - /// (the source was validated when it was parsed) - static std::size_t container_end(const char* src, std::size_t size, std::size_t pos) noexcept - { - const auto* const b = reinterpret_cast(src); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the text as bytes, as the builder reads it - const unsigned char* const e = b + size; - while (pos < size) - { - switch (src[pos]) - { - case ' ': - case '\t': - case '\n': - case '\r': - case ',': - ++pos; - break; - case ']': - case '}': - return pos + 1; - case '/': - if (pos + 1 < size && src[pos + 1] == '/') - { - pos = static_cast(skip_line_comment(b + pos + 2, e) - b); - break; - } - if (pos + 1 < size && src[pos + 1] == '*') - { - // (unterminated: pos is size, which ends the loop) - pos = static_cast(find_block_comment_end(b + pos + 2, e) - b) + 2; - break; - } - return std::string::npos; - default: - return std::string::npos; - } - } - return std::string::npos; - } - - /// the range of the scalar n of document d - void set_scalar(const document_data& d, const node& n) noexcept - { - if ((n.flags & node_flags::is_new) != 0) - { - clear(); - return; - } - std::size_t first = std::string::npos; - std::size_t last = std::string::npos; - switch (n.kind) - { - case node_kind::string: - if ((n.flags & node_flags::storage) == 0) - { - // off is the first byte after the opening quote - if (n.off != 0) - { - first = n.off - 1; - last = n.off + std::size_t{n.len} + 1; - } - } - else if ((n.flags & node_flags::storage) == node_flags::escaped && n.next != 0) - { - // next is the offset of the opening quote plus one - first = n.next - 1; - last = string_end(d.base[0], d.size, first); - } - break; - case node_kind::number_integer: - case node_kind::number_unsigned: - case node_kind::number_float: - if ((n.flags & node_flags::storage) == 0) - { - first = n.off; - last = n.off + number_length(n); - } - break; - case node_kind::boolean: - first = n.off; - last = n.off + ((n.flags & node_flags::is_true) != 0 ? 4u : 5u); - break; - case node_kind::null: - first = n.off; - last = n.off + 4u; - break; - case node_kind::object: - case node_kind::array: - case node_kind::link: - default: - break; - } - // (a crafted image can point past the text) - if (last == std::string::npos || last > d.size) - { - clear(); - } - else - { - set(first, last); - } - } -}; -#endif - -/*! -@brief the basic_json value of the subtree at n - -The subtree is replayed into the SAX handler that parse() uses to build its -values, so the result is the value parse() would produce: duplicate keys keep -the last value, and with JSON_DIAGNOSTICS the parent pointers are set. It is -iterative, so the nesting depth is limited by memory only, as for parse(). - -With JSON_DIAGNOSTIC_POSITIONS, the values get their source positions as with -parse(): position_source stands in for the lexer. Values written by edits have -none (npos), and neither has an array or object whose elements changed, or -whose last element has none (the end of its closing bracket is found by -scanning from the end of the last element). -*/ -template -BasicJsonType materialize(const document_data& d, const node* n) -{ - using string_t = typename BasicJsonType::string_t; - using input_t = iterator_input_adapter; -#if JSON_DIAGNOSTIC_POSITIONS - using lexer_t = position_source; -#else - using lexer_t = lexer; // (the default: unused, as no lexer is passed) -#endif - using sax_t = json_sax_dom_parser; - using nav = navigation; - - // the open containers: their next element (or key of the next member), - // their end, and whether they are objects; for the first 16 levels on the - // stack, else sized once from the document's deepest nesting (for a - // subtree an upper bound) - struct frame - { - const node* pos; - const node* end; - bool object; -#if JSON_DIAGNOSTIC_POSITIONS - std::size_t start; ///< the opening bracket (npos: unknown) - std::size_t last_end; ///< the end of the last element read (npos: unknown) -#endif - }; - -#if !JSON_DIAGNOSTIC_POSITIONS - // a scalar needs neither the handler nor the stack of open containers - // (with positions, it goes through the handler, which sets them) - switch (static_cast(n->kind)) - { - case value_t::string: - return BasicJsonType(string_t(d.str(*n), n->len)); - case value_t::number_integer: - return BasicJsonType(static_cast(static_cast(integer_bits(*n)))); - case value_t::number_unsigned: - return BasicJsonType(static_cast(integer_bits(*n))); - case value_t::number_float: - return BasicJsonType(float_value(d, *n)); - case value_t::boolean: - return BasicJsonType((n->flags & node_flags::is_true) != 0); - case value_t::null: - return BasicJsonType(nullptr); - case value_t::object: - case value_t::array: - case value_t::binary: - case value_t::discarded: - default: - break; - } -#endif - - BasicJsonType result; -#if JSON_DIAGNOSTIC_POSITIONS - position_source positions; - sax_t sax(result, true, &positions); -#else - sax_t sax(result, true); -#endif - const string_t no_token {}; - // (shallow subtrees on the stack: a call for a small value then allocates only for the value) - std::array shallow; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): filled as containers open - std::vector deep{}; - frame* open = shallow.data(); - std::size_t capacity = shallow.size(); - if (d.max_depth > capacity) - { - deep.resize(d.max_depth); - open = deep.data(); - capacity = deep.size(); - } - std::size_t depth = 0; - for (;;) - { -#if JSON_DIAGNOSTIC_POSITIONS - if (!is_container(*n)) - { - positions.set_scalar(d, *n); - if (depth != 0) - { - open[depth - 1].last_end = positions.end; - } - } -#endif - // false positive: n comes from nav::value(), which never returns null for a valid index - // @infer-ignore NULLPTR_DEREFERENCE - switch (static_cast(n->kind)) - { - case value_t::object: - case value_t::array: - { - const bool object = n->kind == node_kind::object; -#if JSON_DIAGNOSTIC_POSITIONS - // (moved containers are also new) - if ((n->flags & node_flags::is_new) != 0 || n->off >= d.size) - { - positions.clear(); - } - else - { - positions.set(std::size_t{n->off}, std::size_t{n->off} + 1); - } -#endif - if (object) - { - sax.start_object(n->len); - } - else - { - sax.start_array(n->len); - } - if (NLOHMANN_VIEW_UNLIKELY(depth == capacity)) - { - // (deeper than the recorded nesting, e.g. after edits) - deep.assign(open, open + depth); - deep.resize(depth * 2); - open = deep.data(); - capacity = deep.size(); - } - // (with positions: until an element is read, the container's - // own bracket is the last thing read) - open[depth++] = frame{nav::first(d, n), nav::end(d, n), object -#if JSON_DIAGNOSTIC_POSITIONS - , positions.start, positions.end -#endif - }; - break; - } - case value_t::string: - { - string_t s(d.str(*n), n->len); - sax.string(s); - break; - } - case value_t::number_integer: - sax.number_integer(static_cast(static_cast(integer_bits(*n)))); - break; - case value_t::number_unsigned: - sax.number_unsigned(static_cast(integer_bits(*n))); - break; - case value_t::number_float: - sax.number_float(float_value(d, *n), no_token); - break; - case value_t::boolean: - sax.boolean((n->flags & node_flags::is_true) != 0); - break; - case value_t::null: - case value_t::binary: - case value_t::discarded: - default: - // false positive: the parser's frame stack points into its own storage (never null) - // @infer-ignore NULLPTR_DEREFERENCE - sax.null(); - break; - } - - // the next value: close finished containers, then read the key - for (;;) - { - if (depth == 0) - { - return result; - } - frame& f = open[depth - 1]; - if (f.pos == f.end) - { -#if JSON_DIAGNOSTIC_POSITIONS - if (f.start == std::string::npos || f.last_end == std::string::npos) - { - positions.clear(); - } - else - { - const std::size_t end = position_source::container_end(d.base[0], d.size, f.last_end); - if (end == std::string::npos) - { - positions.clear(); - } - else - { - // the closing bracket, as the lexer reports it (end_object() - // and end_array() only read its end; the start of the - // container was set when it was opened) - positions.set(end - 1, end); - } - } - if (depth > 1) - { - open[depth - 2].last_end = positions.end; - } -#endif - if (f.object) - { - sax.end_object(); - } - else - { - sax.end_array(); - } - --depth; - continue; - } - const node* entry = f.pos; - if (f.object) - { - string_t key(d.str(*entry), entry->len); - sax.key(key); - ++entry; - } - n = nav::value(entry); - f.pos = document_data::after(entry); - break; - } - } -} - -} // namespace view -} // namespace detail -NLOHMANN_JSON_NAMESPACE_END // #include @@ -8049,6 +8363,7 @@ index_status array_index(const StringType& s, std::size_t& idx) noexcept } /// throws the exception json_pointer::array_index throws for this status +/// (without a context, as json_pointer::array_index) template [[noreturn]] NLOHMANN_VIEW_NOINLINE void throw_array_index_error(index_status status, const StringType& s) { @@ -8088,7 +8403,7 @@ View resolve_pointer(View cur, const Tokens& tokens, pointer_mode mode) { if (mode == pointer_mode::checked) { - throw_key_not_found(token); + throw_key_not_found(site_of(cur), token); } return View(); } @@ -8100,7 +8415,7 @@ View resolve_pointer(View cur, const Tokens& tokens, pointer_mode mode) { if (throwing) { - throw_array_index_past_end(cur.size()); + throw_array_index_past_end(site_of(cur), cur.size()); } return View(); } @@ -8119,7 +8434,7 @@ View resolve_pointer(View cur, const Tokens& tokens, pointer_mode mode) { if (mode == pointer_mode::checked) { - throw_array_index_out_of_range(idx); + throw_array_index_out_of_range(site_of(cur), idx); } return View(); } @@ -8129,7 +8444,7 @@ View resolve_pointer(View cur, const Tokens& tokens, pointer_mode mode) { if (throwing) { - throw_out_of_range(exception_id::pointer_unresolved, concat("unresolved reference token '", token, "'")); + throw_out_of_range(site_of(cur), exception_id::pointer_unresolved, concat("unresolved reference token '", token, "'")); } return View(); } @@ -9779,7 +10094,7 @@ NLOHMANN_VIEW_ALWAYS_INLINE T arithmetic_value(const document_data& d, const nod case value_t::binary: case value_t::discarded: default: - throw_type_error(exception_id::type_mismatch, "type must be number, but is ", value_type_name(static_cast(n.kind))); + throw_type_error(error_site(&d, &n), exception_id::type_mismatch, "type must be number, but is ", value_type_name(static_cast(n.kind))); } } @@ -9789,7 +10104,7 @@ std::vector vector_value(const View& v) { if (NLOHMANN_VIEW_UNLIKELY(!v.is_array())) { - throw_type_error(exception_id::type_mismatch, "type must be array, but is ", v.type_name()); + throw_type_error(site_of(v), exception_id::type_mismatch, "type must be array, but is ", v.type_name()); } std::vector r; r.reserve(v.size()); @@ -9807,7 +10122,7 @@ Map map_value(const View& v) { if (NLOHMANN_VIEW_UNLIKELY(!v.is_object())) { - throw_type_error(exception_id::type_mismatch, "type must be object, but is ", v.type_name()); + throw_type_error(site_of(v), exception_id::type_mismatch, "type must be object, but is ", v.type_name()); } Map r; for (auto it = v.begin(); it != v.end(); ++it) @@ -10064,7 +10379,7 @@ class basic_json_view { return {}; } - detail::view::throw_type_error(detail::exception_id::subscript_wrong_type, "cannot use operator[] with a string argument with ", type_name()); + detail::view::throw_type_error(site(), detail::exception_id::subscript_wrong_type, "cannot use operator[] with a string argument with ", type_name()); } return lookup(key); } @@ -10090,7 +10405,7 @@ class basic_json_view { return {}; } - detail::view::throw_type_error(detail::exception_id::subscript_wrong_type, "cannot use operator[] with a numeric argument with ", type_name()); + detail::view::throw_type_error(site(), detail::exception_id::subscript_wrong_type, "cannot use operator[] with a numeric argument with ", type_name()); } return idx < m_node->len ? element(idx) : basic_json_view(); } @@ -10123,12 +10438,12 @@ class basic_json_view { if (NLOHMANN_VIEW_UNLIKELY(!is_object())) { - detail::view::throw_type_error(detail::exception_id::at_wrong_type, "cannot use at() with ", type_name()); + detail::view::throw_type_error(site(), detail::exception_id::at_wrong_type, "cannot use at() with ", type_name()); } const basic_json_view r = lookup(key); if (NLOHMANN_VIEW_UNLIKELY(r.is_discarded())) { - detail::view::throw_key_not_found(std::string(key.data(), key.size())); + detail::view::throw_key_not_found(site(), std::string(key.data(), key.size())); } return r; } @@ -10149,11 +10464,11 @@ class basic_json_view { if (NLOHMANN_VIEW_UNLIKELY(!is_array())) { - detail::view::throw_type_error(detail::exception_id::at_wrong_type, "cannot use at() with ", type_name()); + detail::view::throw_type_error(site(), detail::exception_id::at_wrong_type, "cannot use at() with ", type_name()); } if (NLOHMANN_VIEW_UNLIKELY(idx >= m_node->len)) { - detail::view::throw_array_index_out_of_range(idx); + detail::view::throw_array_index_out_of_range(site(), idx); } return element(idx); } @@ -10181,7 +10496,7 @@ class basic_json_view { if (NLOHMANN_VIEW_UNLIKELY(!is_object())) { - detail::view::throw_type_error(detail::exception_id::value_wrong_type, "cannot use value() with ", type_name()); + detail::view::throw_type_error(site(), detail::exception_id::value_wrong_type, "cannot use value() with ", type_name()); } const basic_json_view r = lookup(key); return r.is_discarded() ? default_value : r.template get(); @@ -10191,7 +10506,7 @@ class basic_json_view { if (NLOHMANN_VIEW_UNLIKELY(!is_object())) { - detail::view::throw_type_error(detail::exception_id::value_wrong_type, "cannot use value() with ", type_name()); + detail::view::throw_type_error(site(), detail::exception_id::value_wrong_type, "cannot use value() with ", type_name()); } // (the default is made only if there is no such member) const basic_json_view r = lookup(key); @@ -10206,7 +10521,7 @@ class basic_json_view { if (NLOHMANN_VIEW_UNLIKELY(!is_structured())) { - detail::view::throw_type_error(detail::exception_id::value_wrong_type, "cannot use value() with ", type_name()); + detail::view::throw_type_error(site(), detail::exception_id::value_wrong_type, "cannot use value() with ", type_name()); } const basic_json_view r = detail::view::resolve_pointer(*this, detail::json_pointer_access::reference_tokens(ptr), detail::view::pointer_mode::value); return r.is_discarded() ? default_value : r.template get(); @@ -10216,7 +10531,7 @@ class basic_json_view { if (NLOHMANN_VIEW_UNLIKELY(!is_structured())) { - detail::view::throw_type_error(detail::exception_id::value_wrong_type, "cannot use value() with ", type_name()); + detail::view::throw_type_error(site(), detail::exception_id::value_wrong_type, "cannot use value() with ", type_name()); } const basic_json_view r = detail::view::resolve_pointer(*this, detail::json_pointer_access::reference_tokens(ptr), detail::view::pointer_mode::value); return r.is_discarded() ? string_t(default_value) : r.template get(); @@ -10229,7 +10544,7 @@ class basic_json_view const iterator it = begin(); if (NLOHMANN_VIEW_UNLIKELY(it == end())) { - detail::view::throw_invalid_iterator(detail::exception_id::iterator_value_unavailable, "cannot get value"); + detail::view::throw_invalid_iterator(site(), detail::exception_id::iterator_value_unavailable, "cannot get value"); } return *it; } @@ -10385,7 +10700,7 @@ class basic_json_view { if (NLOHMANN_VIEW_UNLIKELY(!is_string())) { - detail::view::throw_type_error(detail::exception_id::type_mismatch, "type must be string, but is ", type_name()); + detail::view::throw_type_error(site(), detail::exception_id::type_mismatch, "type must be string, but is ", type_name()); } return {m_doc->str(*m_node), m_node->len}; } @@ -10397,7 +10712,7 @@ class basic_json_view { if (NLOHMANN_VIEW_UNLIKELY(!is_number())) { - detail::view::throw_type_error(detail::exception_id::type_mismatch, "type must be number, but is ", type_name()); + detail::view::throw_type_error(site(), detail::exception_id::type_mismatch, "type must be number, but is ", type_name()); } return {m_doc->str(*m_node), detail::view::number_length(*m_node)}; } @@ -10543,12 +10858,19 @@ class basic_json_view template friend class basic_json_view; template friend class detail::view::editor; template friend class detail::view::view_side; + friend struct detail::view::view_access; friend iterator; basic_json_view(const document_data* d, const node* n) noexcept : m_doc(d), m_node(n) {} + /// this value as the context of an exception + detail::view::error_site site() const noexcept + { + return {m_doc, m_node}; + } + using json_side_t = detail::view::json_side; static detail::view::view_side side(const basic_json_view& v) noexcept @@ -10655,7 +10977,7 @@ class basic_json_view { if (NLOHMANN_VIEW_UNLIKELY(!is_boolean())) { - detail::view::throw_type_error(detail::exception_id::type_mismatch, "type must be boolean, but is ", type_name()); + detail::view::throw_type_error(site(), detail::exception_id::type_mismatch, "type must be boolean, but is ", type_name()); } return (m_node->flags & detail::view::node_flags::is_true) != 0; } @@ -10665,7 +10987,7 @@ class basic_json_view { if (NLOHMANN_VIEW_UNLIKELY(m_node == nullptr)) { - detail::view::throw_type_error(detail::exception_id::type_mismatch, "type must be number, but is ", type_name()); + detail::view::throw_type_error(site(), detail::exception_id::type_mismatch, "type must be number, but is ", type_name()); } return detail::view::arithmetic_value(*m_doc, *m_node); } @@ -10674,7 +10996,7 @@ class basic_json_view { if (NLOHMANN_VIEW_UNLIKELY(!is_null())) { - detail::view::throw_type_error(detail::exception_id::type_mismatch, "type must be null, but is ", type_name()); + detail::view::throw_type_error(site(), detail::exception_id::type_mismatch, "type must be null, but is ", type_name()); } return nullptr; } @@ -11089,7 +11411,7 @@ class basic_json_document template::value, int>::type = 0> view_type set(view_type array, I idx, V && value) { - return editor().set(array, index(idx), std::forward(value)); + return editor().set(array, index(array, idx), std::forward(value)); } /// set the value at a JSON pointer: its parent must exist; an object @@ -11126,7 +11448,7 @@ class basic_json_document const std::size_t idx = pointer_index(token); if (idx >= 0xFFFFFFFFu) { - detail::view::throw_array_index_out_of_range(idx); + detail::view::throw_array_index_out_of_range(detail::view::site_of(parent), idx); } for (std::size_t i = 0; i < idx; ++i) { @@ -11160,7 +11482,7 @@ class basic_json_document template::value, int>::type = 0> view_type insert(view_type array, I idx, V && value) { - return editor().insert(array, index(idx), std::forward(value)); + return editor().insert(array, index(array, idx), std::forward(value)); } /// remove all members with this key; returns their number @@ -11173,7 +11495,7 @@ class basic_json_document template::value, int>::type = 0> void erase(view_type array, I idx) { - editor().erase(array, index(idx)); + editor().erase(array, index(array, idx)); } /// remove the value at a JSON pointer; returns the number of removed @@ -11207,25 +11529,25 @@ class basic_json_document return detail::view::editor(*m_data); } - /// an index (out_of_range.401 if negative) + /// an index into @a array (out_of_range.401 if negative) template - static std::size_t index(I idx) + static std::size_t index(const view_type& array, I idx) { - return index(idx, std::is_signed {}); + return index(array, idx, std::is_signed {}); } template - static std::size_t index(I idx, std::true_type /*signed*/) + static std::size_t index(const view_type& array, I idx, std::true_type /*signed*/) { if (idx < 0) { - detail::view::throw_array_index_out_of_range(idx); + detail::view::throw_array_index_out_of_range(detail::view::site_of(array), idx); } return static_cast(idx); } template - static std::size_t index(I idx, std::false_type /*unsigned*/) + static std::size_t index(const view_type& /*array*/, I idx, std::false_type /*unsigned*/) { return static_cast(idx); } diff --git a/tests/src/json_view_test_helpers.hpp b/tests/src/json_view_test_helpers.hpp index b5af61798..87822815b 100644 --- a/tests/src/json_view_test_helpers.hpp +++ b/tests/src/json_view_test_helpers.hpp @@ -15,6 +15,7 @@ #include #include // any_of +#include // size_t #include // function #include // mt19937 #include // string @@ -88,6 +89,27 @@ inline bool has_duplicate_keys(const nlohmann::ordered_json_view& v) }); } +// the message of an exception about the value at path (a JSON pointer, "" for +// the root) whose source bytes are [start, end): the context basic_json adds +// with JSON_DIAGNOSTICS and JSON_DIAGNOSTIC_POSITIONS, if enabled, then what +inline std::string message(const std::string& id, const std::string& path, std::size_t start, std::size_t end, const std::string& what) +{ +#if JSON_DIAGNOSTIC_POSITIONS + const std::string positions = "(bytes " + std::to_string(start) + "-" + std::to_string(end) + ") "; +#else + const std::string positions; + static_cast(start); + static_cast(end); +#endif +#if JSON_DIAGNOSTICS + const std::string context = path.empty() ? "" : "(" + path + ") " + positions; +#else + const std::string& context = positions; + static_cast(path); +#endif + return "[json.exception." + id + "] " + context + what; +} + #if !defined(JSON_NOEXCEPTION) // the exception a call throws, or "" if it throws none inline std::string exception_of_call(const std::function& f) diff --git a/tests/src/unit-json_view.cpp b/tests/src/unit-json_view.cpp index 77f7aa4fa..c823ef9e2 100644 --- a/tests/src/unit-json_view.cpp +++ b/tests/src/unit-json_view.cpp @@ -225,7 +225,7 @@ TEST_CASE("json_view") #if JSON_DIAGNOSTICS // the parents are set, so errors name the path const json m = materialized(R"({"a":{"b":[1]}})"); - CHECK_THROWS_WITH_AS(m.at("a").at("b").at(0).at("x"), "[json.exception.type_error.304] (/a/b/0) cannot use at() with number", json::type_error&); + CHECK_THROWS_WITH_AS(m.at("a").at("b").at(0).at("x"), json_view_test::message("type_error.304", "/a/b/0", 11, 12, "cannot use at() with number").c_str(), json::type_error&); #endif } @@ -750,19 +750,6 @@ std::string exception_of(F f) return ""; } -// the exception a call on a basic_json value throws, without the byte range of -// the value (JSON_DIAGNOSTIC_POSITIONS), which the exceptions of views lack -template -std::string json_exception_of(F f) -{ - std::string what = exception_of(f); - const std::size_t at = what.find("] (bytes "); - if (at != std::string::npos) - { - what.erase(at + 2, what.find(") ", at) - at); - } - return what; -} #endif // compares a view with the ordered_json value materialize() gives for it: @@ -971,24 +958,25 @@ TEST_CASE("json_view element access and iteration") #if !defined(JSON_NOEXCEPTION) if (!j.is_object()) { - CHECK(exception_of([&] { static_cast(v["a"]); }) == json_exception_of([&] { static_cast(j["a"]); })); + CHECK(exception_of([&] { static_cast(v["a"]); }) == exception_of([&] { static_cast(j["a"]); })); } if (!j.is_array()) { - CHECK(exception_of([&] { static_cast(v[0]); }) == json_exception_of([&] { static_cast(j[0]); })); + CHECK(exception_of([&] { static_cast(v[0]); }) == exception_of([&] { static_cast(j[0]); })); } - CHECK(exception_of([&] { static_cast(v.at("a")); }) == json_exception_of([&] { static_cast(j.at("a")); })); - CHECK(exception_of([&] { static_cast(v.at("missing")); }) == json_exception_of([&] { static_cast(j.at("missing")); })); - CHECK(exception_of([&] { static_cast(v.at(0)); }) == json_exception_of([&] { static_cast(j.at(0)); })); - CHECK(exception_of([&] { static_cast(v.at(5)); }) == json_exception_of([&] { static_cast(j.at(5)); })); + CHECK(exception_of([&] { static_cast(v.at("a")); }) == exception_of([&] { static_cast(j.at("a")); })); + CHECK(exception_of([&] { static_cast(v.at("missing")); }) == exception_of([&] { static_cast(j.at("missing")); })); + CHECK(exception_of([&] { static_cast(v.at(0)); }) == exception_of([&] { static_cast(j.at(0)); })); + CHECK(exception_of([&] { static_cast(v.at(5)); }) == exception_of([&] { static_cast(j.at(5)); })); if (!(j.is_object() && j.empty())) // (key() of an end iterator) { - CHECK(exception_of([&] { static_cast(v.begin().key()); }) == json_exception_of([&] { static_cast(j.begin().key()); })); + // (without the context of basic_json: view iterators do not know the value they iterate over) + CHECK(exception_of([&] { static_cast(v.begin().key()); }) == (j.is_object() ? "" : "[json.exception.invalid_iterator.207] cannot use key() for non-object iterators")); } if (!j.empty() || j.is_null()) { - CHECK(exception_of([&] { static_cast(v.front()); }) == json_exception_of([&] { static_cast(j.front()); })); - CHECK(exception_of([&] { static_cast(v.back()); }) == json_exception_of([&] { static_cast(j.back()); })); + CHECK(exception_of([&] { static_cast(v.front()); }) == exception_of([&] { static_cast(j.front()); })); + CHECK(exception_of([&] { static_cast(v.back()); }) == exception_of([&] { static_cast(j.back()); })); } #endif CHECK(v.contains("a") == j.contains("a")); @@ -1000,8 +988,8 @@ TEST_CASE("json_view element access and iteration") const json_document d = json_document::parse(R"({"a":[]})"); CHECK(d.root()["b"].is_discarded()); CHECK(d.root()["a"][0].is_discarded()); - CHECK_THROWS_WITH_AS(d.root()["a"].front(), "[json.exception.invalid_iterator.214] cannot get value", json::invalid_iterator&); - CHECK_THROWS_WITH_AS(d.root()["a"].back(), "[json.exception.invalid_iterator.214] cannot get value", json::invalid_iterator&); + CHECK_THROWS_WITH_AS(d.root()["a"].front(), json_view_test::message("invalid_iterator.214", "/a", 5, 7, "cannot get value").c_str(), json::invalid_iterator&); + CHECK_THROWS_WITH_AS(d.root()["a"].back(), json_view_test::message("invalid_iterator.214", "/a", 5, 7, "cannot get value").c_str(), json::invalid_iterator&); const json_view invalid{}; CHECK(invalid.begin() == invalid.end()); CHECK(std::string(invalid.type_name()) == "discarded"); @@ -1032,10 +1020,10 @@ TEST_CASE("json_view element access and iteration") CHECK(v[json_view::json_pointer("/a/b/5")][0].is_discarded()); #if !defined(JSON_NOEXCEPTION) // type errors on values that are not discarded stay - CHECK_THROWS_WITH_AS(v[0], "[json.exception.type_error.305] cannot use operator[] with a numeric argument with object", json::type_error&); - CHECK_THROWS_WITH_AS(v["a"]["b"]["c"], "[json.exception.type_error.305] cannot use operator[] with a string argument with array", json::type_error&); - CHECK_THROWS_WITH_AS(v["s"]["c"], "[json.exception.type_error.305] cannot use operator[] with a string argument with string", json::type_error&); - CHECK_THROWS_WITH_AS(v["s"][0], "[json.exception.type_error.305] cannot use operator[] with a numeric argument with string", json::type_error&); + CHECK_THROWS_WITH_AS(v[0], json_view_test::message("type_error.305", "", 0, 29, "cannot use operator[] with a numeric argument with object").c_str(), json::type_error&); + CHECK_THROWS_WITH_AS(v["a"]["b"]["c"], json_view_test::message("type_error.305", "/a/b", 10, 17, "cannot use operator[] with a string argument with array").c_str(), json::type_error&); + CHECK_THROWS_WITH_AS(v["s"]["c"], json_view_test::message("type_error.305", "/s", 23, 28, "cannot use operator[] with a string argument with string").c_str(), json::type_error&); + CHECK_THROWS_WITH_AS(v["s"][0], json_view_test::message("type_error.305", "/s", 23, 28, "cannot use operator[] with a numeric argument with string").c_str(), json::type_error&); CHECK_THROWS_AS(v["a"]["b"][0]["c"], json::type_error&); CHECK_THROWS_AS(v["s"][json_view::json_pointer("/x")], json::out_of_range&); // at() keeps throwing on a discarded view @@ -1110,9 +1098,9 @@ TEST_CASE("json_view element access and iteration") CHECK(v[(std::numeric_limits::max)()].is_discarded()); CHECK(v[std::numeric_limits::max()].is_discarded()); #if !defined(JSON_NOEXCEPTION) - CHECK_THROWS_WITH_AS(v.at(3), "[json.exception.out_of_range.401] array index 3 is out of range", json::out_of_range&); - CHECK_THROWS_WITH_AS(v.at(3u), "[json.exception.out_of_range.401] array index 3 is out of range", json::out_of_range&); - CHECK_THROWS_WITH_AS(v.at(std::int64_t(3)), "[json.exception.out_of_range.401] array index 3 is out of range", json::out_of_range&); + CHECK_THROWS_WITH_AS(v.at(3), json_view_test::message("out_of_range.401", "", 0, 10, "array index 3 is out of range").c_str(), json::out_of_range&); + CHECK_THROWS_WITH_AS(v.at(3u), json_view_test::message("out_of_range.401", "", 0, 10, "array index 3 is out of range").c_str(), json::out_of_range&); + CHECK_THROWS_WITH_AS(v.at(std::int64_t(3)), json_view_test::message("out_of_range.401", "", 0, 10, "array index 3 is out of range").c_str(), json::out_of_range&); CHECK_THROWS_AS(v.at(-1), json::out_of_range&); CHECK_THROWS_AS(v.at(-1L), json::out_of_range&); CHECK_THROWS_AS(v.at(std::int64_t(-1)), json::out_of_range&); @@ -1246,26 +1234,6 @@ TEST_CASE("json_view element access and iteration") namespace { -#if !defined(JSON_NOEXCEPTION) -// an exception message without the context that basic_json adds with -// JSON_DIAGNOSTICS ("(/path) ") and JSON_DIAGNOSTIC_POSITIONS ("(bytes 1-2) "); -// the view's exceptions have no such context -std::string without_path(std::string msg) -{ - for (const char* prefix : - {"] (/", "] (bytes " - }) - { - const std::size_t open = msg.find(prefix); - if (open != std::string::npos) - { - msg.erase(open + 2, msg.find(") ", open) + 2 - (open + 2)); - } - } - return msg; -} -#endif - // the bits of a float, to compare values bit for bit std::uint64_t bits(double x) { @@ -1336,18 +1304,18 @@ void check_values(const ordered_json_view& v, const ordered_json& j, const std:: // conversions to the wrong type throw what basic_json throws if (!j.is_number()) { - CHECK(exception_of([&] { static_cast(v.get()); }) == without_path(exception_of([&] { static_cast(j.get()); }))); + CHECK(exception_of([&] { static_cast(v.get()); }) == exception_of([&] { static_cast(j.get()); })); } - CHECK(exception_of([&] { static_cast(v.get()); }) == without_path(exception_of([&] { static_cast(j.get()); }))); - CHECK(exception_of([&] { static_cast(v.get()); }) == without_path(exception_of([&] { static_cast(j.get()); }))); - CHECK(exception_of([&] { static_cast(v.get()); }) == without_path(exception_of([&] { static_cast(j.get()); }))); + CHECK(exception_of([&] { static_cast(v.get()); }) == exception_of([&] { static_cast(j.get()); })); + CHECK(exception_of([&] { static_cast(v.get()); }) == exception_of([&] { static_cast(j.get()); })); + CHECK(exception_of([&] { static_cast(v.get()); }) == exception_of([&] { static_cast(j.get()); })); if (!j.is_array()) { - CHECK(exception_of([&] { static_cast(v.get>()); }) == without_path(exception_of([&] { static_cast(j.get>()); }))); + CHECK(exception_of([&] { static_cast(v.get>()); }) == exception_of([&] { static_cast(j.get>()); })); } if (!j.is_object()) { - CHECK(exception_of([&] { static_cast(v.get>()); }) == without_path(exception_of([&] { static_cast(j.get>()); }))); + CHECK(exception_of([&] { static_cast(v.get>()); }) == exception_of([&] { static_cast(j.get>()); })); } #endif @@ -1490,8 +1458,8 @@ TEST_CASE("json_view values") CHECK(v[3].number_token() == "123456789012345678901234567890"); CHECK(v[4].number_token() == "-12"); CHECK(v[5].number_token() == "7"); - CHECK_THROWS_WITH_AS(v[6].number_token(), "[json.exception.type_error.302] type must be number, but is string", json::type_error&); - CHECK_THROWS_WITH_AS(v.get_string(), "[json.exception.type_error.302] type must be string, but is array", json::type_error&); + CHECK_THROWS_WITH_AS(v[6].number_token(), json_view_test::message("type_error.302", "/6", 56, 59, "type must be number, but is string").c_str(), json::type_error&); + CHECK_THROWS_WITH_AS(v.get_string(), json_view_test::message("type_error.302", "", 0, 60, "type must be string, but is array").c_str(), json::type_error&); } SECTION("conversions") @@ -1565,15 +1533,15 @@ TEST_CASE("json_view values") CHECK(v["o"]["x"].value(json::json_pointer("/1"), 0) == j["o"]["x"].value(json::json_pointer("/1"), 0)); CHECK(v["o"]["x"].value(json::json_pointer("/7"), 3) == j["o"]["x"].value(json::json_pointer("/7"), 3)); #if !defined(JSON_NOEXCEPTION) - CHECK(exception_of([&] { static_cast(v["o"]["x"].value("k", 0)); }) == without_path(exception_of([&] { static_cast(j["o"]["x"].value("k", 0)); }))); - CHECK(exception_of([&] { static_cast(v.value("s", 0)); }) == without_path(exception_of([&] { static_cast(j.value("s", 0)); }))); - CHECK(exception_of([&] { static_cast(v["n"].value("x", 0)); }) == without_path(exception_of([&] { static_cast(j["n"].value("x", 0)); }))); - CHECK(exception_of([&] { static_cast(v["n"].value(json::json_pointer("/x"), 0)); }) == without_path(exception_of([&] { static_cast(j["n"].value(json::json_pointer("/x"), 0)); }))); + CHECK(exception_of([&] { static_cast(v["o"]["x"].value("k", 0)); }) == exception_of([&] { static_cast(j["o"]["x"].value("k", 0)); })); + CHECK(exception_of([&] { static_cast(v.value("s", 0)); }) == exception_of([&] { static_cast(j.value("s", 0)); })); + CHECK(exception_of([&] { static_cast(v["n"].value("x", 0)); }) == exception_of([&] { static_cast(j["n"].value("x", 0)); })); + CHECK(exception_of([&] { static_cast(v["n"].value(json::json_pointer("/x"), 0)); }) == exception_of([&] { static_cast(j["n"].value(json::json_pointer("/x"), 0)); })); // string defaults: the member is not a string, or the value is neither an object nor an array - CHECK(exception_of([&] { static_cast(v.value("n", "default")); }) == without_path(exception_of([&] { static_cast(j.value("n", "default")); }))); - CHECK(exception_of([&] { static_cast(v["n"].value("x", "default")); }) == without_path(exception_of([&] { static_cast(j["n"].value("x", "default")); }))); - CHECK(exception_of([&] { static_cast(v.value(json::json_pointer("/n"), "default")); }) == without_path(exception_of([&] { static_cast(j.value(json::json_pointer("/n"), "default")); }))); - CHECK(exception_of([&] { static_cast(v["s"].value(json::json_pointer("/x"), "default")); }) == without_path(exception_of([&] { static_cast(j["s"].value(json::json_pointer("/x"), "default")); }))); + CHECK(exception_of([&] { static_cast(v.value("n", "default")); }) == exception_of([&] { static_cast(j.value("n", "default")); })); + CHECK(exception_of([&] { static_cast(v["n"].value("x", "default")); }) == exception_of([&] { static_cast(j["n"].value("x", "default")); })); + CHECK(exception_of([&] { static_cast(v.value(json::json_pointer("/n"), "default")); }) == exception_of([&] { static_cast(j.value(json::json_pointer("/n"), "default")); })); + CHECK(exception_of([&] { static_cast(v["s"].value(json::json_pointer("/x"), "default")); }) == exception_of([&] { static_cast(j["s"].value(json::json_pointer("/x"), "default")); })); #endif // (defaults and members longer than the short string optimization) CHECK(v.value("s", "a default longer than fifteen characters") == "text"); @@ -1632,7 +1600,7 @@ TEST_CASE("json_view JSON pointers") { CAPTURE(pointer) const json::json_pointer p(pointer); - const std::string at_error = without_path(exception_of([&] { static_cast(j.at(p)); })); + const std::string at_error = exception_of([&] { static_cast(j.at(p)); }); CHECK(exception_of([&] { static_cast(v.at(p)); }) == at_error); if (at_error.empty()) { @@ -1646,7 +1614,7 @@ TEST_CASE("json_view JSON pointers") } else { - CHECK(exception_of([&] { static_cast(v[p]); }) == without_path(exception_of([&] { static_cast(j[p]); }))); + CHECK(exception_of([&] { static_cast(v[p]); }) == exception_of([&] { static_cast(j[p]); })); } // (basic_json::contains() throws out_of_range.404 for an empty // array index token, although it is not meant to throw; the view @@ -1657,7 +1625,7 @@ TEST_CASE("json_view JSON pointers") static_cast(found); }); CHECK(v.contains(p) == (contains_error.empty() && j.contains(p))); - CHECK(exception_of([&] { static_cast(v.value(p, 5)); }) == without_path(exception_of([&] { static_cast(j.value(p, 5)); }))); + CHECK(exception_of([&] { static_cast(v.value(p, 5)); }) == exception_of([&] { static_cast(j.value(p, 5)); })); } } #endif diff --git a/tests/src/unit-json_view_edit.cpp b/tests/src/unit-json_view_edit.cpp index 1b1a24ec8..2cc3d4701 100644 --- a/tests/src/unit-json_view_edit.cpp +++ b/tests/src/unit-json_view_edit.cpp @@ -532,18 +532,18 @@ TEST_CASE("json_view edits: errors") CHECK_THROWS_WITH_AS(d.set(other_editable.root(), 1), "[json.exception.invalid_iterator.202] view does not belong to this document", json::invalid_iterator&); CHECK_THROWS_WITH_AS(d.set(json_editable_view(), 1), "[json.exception.invalid_iterator.202] view does not belong to this document", json::invalid_iterator&); - CHECK_THROWS_WITH_AS(d.set(root["a"], "k", 1), "[json.exception.type_error.305] cannot use operator[] with a string argument with array", json::type_error&); - CHECK_THROWS_WITH_AS(d.set(root["o"], 0, 1), "[json.exception.type_error.305] cannot use operator[] with a numeric argument with object", json::type_error&); - CHECK_THROWS_WITH_AS(d.set(root["a"], 2, 1), "[json.exception.out_of_range.401] array index 2 is out of range", json::out_of_range&); - CHECK_THROWS_WITH_AS(d.set(root["a"], -1, 1), "[json.exception.out_of_range.401] array index -1 is out of range", json::out_of_range&); - CHECK_THROWS_WITH_AS(d.push_back(root["o"], 1), "[json.exception.type_error.308] cannot use push_back() with object", json::type_error&); - CHECK_THROWS_WITH_AS(d.insert(root["n"], 0, 1), "[json.exception.type_error.309] cannot use insert() with number", json::type_error&); - CHECK_THROWS_WITH_AS(d.insert(root["a"], 3, 1), "[json.exception.out_of_range.401] array index 3 is out of range", json::out_of_range&); - CHECK_THROWS_WITH_AS(d.erase(root["n"], "k"), "[json.exception.type_error.307] cannot use erase() with number", json::type_error&); - CHECK_THROWS_WITH_AS(d.erase(root["o"], 0), "[json.exception.type_error.307] cannot use erase() with object", json::type_error&); - CHECK_THROWS_WITH_AS(d.erase(root["a"], 2), "[json.exception.out_of_range.401] array index 2 is out of range", json::out_of_range&); + CHECK_THROWS_WITH_AS(d.set(root["a"], "k", 1), json_view_test::message("type_error.305", "/a", 21, 27, "cannot use operator[] with a string argument with array").c_str(), json::type_error&); + CHECK_THROWS_WITH_AS(d.set(root["o"], 0, 1), json_view_test::message("type_error.305", "/o", 6, 14, "cannot use operator[] with a numeric argument with object").c_str(), json::type_error&); + CHECK_THROWS_WITH_AS(d.set(root["a"], 2, 1), json_view_test::message("out_of_range.401", "/a", 21, 27, "array index 2 is out of range").c_str(), json::out_of_range&); + CHECK_THROWS_WITH_AS(d.set(root["a"], -1, 1), json_view_test::message("out_of_range.401", "/a", 21, 27, "array index -1 is out of range").c_str(), json::out_of_range&); + CHECK_THROWS_WITH_AS(d.push_back(root["o"], 1), json_view_test::message("type_error.308", "/o", 6, 14, "cannot use push_back() with object").c_str(), json::type_error&); + CHECK_THROWS_WITH_AS(d.insert(root["n"], 0, 1), json_view_test::message("type_error.309", "/n", 34, 35, "cannot use insert() with number").c_str(), json::type_error&); + CHECK_THROWS_WITH_AS(d.insert(root["a"], 3, 1), json_view_test::message("out_of_range.401", "/a", 21, 27, "array index 3 is out of range").c_str(), json::out_of_range&); + CHECK_THROWS_WITH_AS(d.erase(root["n"], "k"), json_view_test::message("type_error.307", "/n", 34, 35, "cannot use erase() with number").c_str(), json::type_error&); + CHECK_THROWS_WITH_AS(d.erase(root["o"], 0), json_view_test::message("type_error.307", "/o", 6, 14, "cannot use erase() with object").c_str(), json::type_error&); + CHECK_THROWS_WITH_AS(d.erase(root["a"], 2), json_view_test::message("out_of_range.401", "/a", 21, 27, "array index 2 is out of range").c_str(), json::out_of_range&); CHECK_THROWS_WITH_AS(d.erase(json::json_pointer("")), "[json.exception.out_of_range.405] JSON pointer has no parent", json::out_of_range&); - CHECK_THROWS_WITH_AS(d.erase(json::json_pointer("/missing/x")), "[json.exception.out_of_range.403] key 'missing' not found", json::out_of_range&); + CHECK_THROWS_WITH_AS(d.erase(json::json_pointer("/missing/x")), json_view_test::message("out_of_range.403", "", 0, 47, "key 'missing' not found").c_str(), json::out_of_range&); CHECK_THROWS_WITH_AS(d.set(json::json_pointer("/a/01"), 1), "[json.exception.parse_error.106] parse error: array index '01' must not begin with '0'", json::parse_error&); CHECK_THROWS_WITH_AS(d.set(root, json_editable_view()), "[json.exception.type_error.302] type must be a value, but is discarded", json::type_error&); CHECK_THROWS_WITH_AS(d.set(root, json::binary({1, 2})), "[json.exception.type_error.319] cannot store a binary value in a json_document", json::type_error&); diff --git a/tests/src/unit-json_view_from_json.cpp b/tests/src/unit-json_view_from_json.cpp index fa01b639d..824585a56 100644 --- a/tests/src/unit-json_view_from_json.cpp +++ b/tests/src/unit-json_view_from_json.cpp @@ -338,8 +338,7 @@ TEST_CASE("json_view from_json") CHECK(d.root().get>().at("a") == 3); } - // (the exceptions of views have neither the path nor the byte range of the value) -#if !defined(JSON_NOEXCEPTION) && !JSON_DIAGNOSTICS && !JSON_DIAGNOSTIC_POSITIONS +#if !defined(JSON_NOEXCEPTION) SECTION("std types: the same errors as from the materialized value") { const json_document d = json_document::parse(R"([[1, "x"], {"a": 1}, "text", [1], 5, null])"); @@ -404,10 +403,10 @@ TEST_CASE("json_view from_json") CHECK(d.root()[4].get() == strict_color::red); #if !defined(JSON_NOEXCEPTION) const json j = d.root().materialize(); - CHECK(exception_of_call([&] { static_cast(d.root()[2].get()); }) == "[json.exception.out_of_range.410] enum value out of range for strict_color: \"purple\""); #if !JSON_DIAGNOSTICS && !JSON_DIAGNOSTIC_POSITIONS - CHECK(exception_of_call([&] { static_cast(d.root()[2].get()); }) == exception_of_call([&] { static_cast(j[2].get()); })); + CHECK(exception_of_call([&] { static_cast(d.root()[2].get()); }) == "[json.exception.out_of_range.410] enum value out of range for strict_color: \"purple\""); #endif + CHECK(exception_of_call([&] { static_cast(d.root()[2].get()); }) == exception_of_call([&] { static_cast(j[2].get()); })); #endif } @@ -477,11 +476,9 @@ TEST_CASE("json_view from_json") CHECK(o.root().get().center.y == 2); } -#if !defined(JSON_NOEXCEPTION) && !JSON_DIAGNOSTICS && !JSON_DIAGNOSTIC_POSITIONS +#if !defined(JSON_NOEXCEPTION) SECTION("errors are those of the same conversion of a json value") { - // (with JSON_DIAGNOSTICS, those of json name the path of the value, - // and with JSON_DIAGNOSTIC_POSITIONS its bytes) const json_document d = json_document::parse(R"([{"x": 1}, {"x": "1", "y": 2}, [1, 2], {"id": "7", "c": "red"}])"); const json j = d.root().materialize(); for (std::size_t i = 0; i < 3; ++i) diff --git a/tests/src/unit-json_view_large.cpp b/tests/src/unit-json_view_large.cpp index 89d7db326..5ad65c812 100644 --- a/tests/src/unit-json_view_large.cpp +++ b/tests/src/unit-json_view_large.cpp @@ -163,7 +163,7 @@ TEST_CASE("json_view large objects") CHECK(v[""].get_string() == "empty key"); CHECK(v["k1"].get() == 1); // the first of duplicate keys, as for small objects CHECK(!v.contains("missing")); - CHECK_THROWS_WITH_AS(v.at("missing"), "[json.exception.out_of_range.403] key 'missing' not found", json::out_of_range&); + CHECK_THROWS_WITH_AS(v.at("missing"), json_view_test::message("out_of_range.403", "", 0, text.size(), "key 'missing' not found").c_str(), json::out_of_range&); CHECK(v == j); CHECK(v.materialize() == j); } diff --git a/tests/src/unit-json_view_positions.cpp b/tests/src/unit-json_view_positions.cpp index 08ea47a08..c6afd7fcb 100644 --- a/tests/src/unit-json_view_positions.cpp +++ b/tests/src/unit-json_view_positions.cpp @@ -81,6 +81,112 @@ std::string range(const json& j) { return j.start_pos() == npos && j.end_pos() == npos ? "none" : std::to_string(j.start_pos()) + "-" + std::to_string(j.end_pos()); } + +#if !defined(JSON_NOEXCEPTION) +using json_view_test::exception_of_call; + +// the exceptions about every value of v are those about the value at the same +// place in j, a value of the whole document from parse() or materialize() (so +// that its parents and positions are set); members whose key repeats are +// skipped (the view finds the first one, j holds the last) +template +void check_exceptions(const View& v, const json& j) +{ + CAPTURE(j.dump()) + if (!v.is_number()) + { + CHECK(exception_of_call([&] { static_cast(v.template get()); }) == exception_of_call([&] { static_cast(j.get()); })); + } + if (!v.is_string()) + { + CHECK(exception_of_call([&] { static_cast(v.template get()); }) == exception_of_call([&] { static_cast(j.get()); })); + } + if (!v.is_array()) + { + CHECK(exception_of_call([&] { static_cast(v.template get>()); }) == exception_of_call([&] { static_cast(j.get>()); })); + } + CHECK(exception_of_call([&] { static_cast(v.at("missing")); }) == exception_of_call([&] { static_cast(j.at("missing")); })); + CHECK(exception_of_call([&] { static_cast(v.at(1000)); }) == exception_of_call([&] { static_cast(j.at(1000)); })); + CHECK(exception_of_call([&] { static_cast(v.at(json::json_pointer("/missing"))); }) == exception_of_call([&] { static_cast(j.at(json::json_pointer("/missing"))); })); + CHECK(exception_of_call([&] { static_cast(v.at(json::json_pointer("/-"))); }) == exception_of_call([&] { static_cast(j.at(json::json_pointer("/-"))); })); + CHECK(exception_of_call([&] { static_cast(v.value("k", 0)); }) == exception_of_call([&] { static_cast(j.value("k", 0)); })); + if (!v.is_object()) + { + CHECK(exception_of_call([&] { static_cast(v["k"]); }) == exception_of_call([&] { static_cast(j["k"]); })); + // (without the context of basic_json: view iterators do not know the value they iterate over) + CHECK(exception_of_call([&] { static_cast(v.begin().key()); }) == "[json.exception.invalid_iterator.207] cannot use key() for non-object iterators"); + } + if (!v.is_array()) + { + CHECK(exception_of_call([&] { static_cast(v[0]); }) == exception_of_call([&] { static_cast(j[0]); })); + } + if (v.is_null()) + { + CHECK(exception_of_call([&] { static_cast(v.front()); }) == exception_of_call([&] { static_cast(j.front()); })); + } + if (v.is_object()) + { + for (auto it = v.begin(); it != v.end(); ++it) + { + const std::string key(it.key().data(), it.key().size()); + if (v.count(key) == 1) + { + check_exceptions(it.value(), j.at(key)); + } + } + } + else if (v.is_array()) + { + for (std::size_t i = 0; i < v.size(); ++i) + { + check_exceptions(v[i], j[i]); + } + } +} + +// as check_exceptions, for edits that fail (they change neither d nor j) +void check_edit_exceptions(json_editable_document& d, const json_editable_document::view_type& v, json& j) +{ + CAPTURE(j.dump()) + if (!v.is_object() && !v.is_null()) + { + CHECK(exception_of_call([&] { d.set(v, "k", 1); }) == exception_of_call([&] { j["k"] = 1; })); + CHECK(exception_of_call([&] { d.erase(v, "k"); }) == exception_of_call([&] { j.erase("k"); })); + } + if (!v.is_array() && !v.is_null()) + { + CHECK(exception_of_call([&] { d.set(v, 0, 1); }) == exception_of_call([&] { j[0] = 1; })); + CHECK(exception_of_call([&] { d.push_back(v, 1); }) == exception_of_call([&] { j.push_back(1); })); + } + if (!v.is_array()) + { + CHECK(exception_of_call([&] { d.insert(v, 0, 1); }) == exception_of_call([&] { j.insert(j.cbegin(), 1); })); + CHECK(exception_of_call([&] { d.erase(v, 0); }) == exception_of_call([&] { j.erase(0); })); + } + else + { + CHECK(exception_of_call([&] { d.erase(v, 1000); }) == exception_of_call([&] { j.erase(1000); })); + } + if (v.is_object()) + { + for (auto it = v.begin(); it != v.end(); ++it) + { + const std::string key(it.key().data(), it.key().size()); + if (v.count(key) == 1) + { + check_edit_exceptions(d, it.value(), j.at(key)); + } + } + } + else if (v.is_array()) + { + for (std::size_t i = 0; i < v.size(); ++i) + { + check_edit_exceptions(d, v[i], j[i]); + } + } +} +#endif } // namespace TEST_CASE("json_view materialize() source positions") @@ -345,3 +451,123 @@ TEST_CASE("json_view materialize() source positions") } #endif } + +#if !defined(JSON_NOEXCEPTION) +TEST_CASE("json_view exceptions name the path and the bytes of the value") +{ + SECTION("as those of basic_json") + { + const std::vector texts = + { + "null", "true", "42", "\"s\"", "[]", "{}", "[1, [2, [3, []]]]", R"({"a": {"b": [1, {"c": null}]}, "d": "e"})", + R"({"a/b": {"~": [1]}, "\u0041\n": {"x": "\t"}, "": [{}]})", + R"([1 /* ] */, {"a": [] // } +, "b": {} } , ])", + "\n\t[\r\n 1 ,\n {\n \"a\" : [ ]\n }\n]\n", + }; + for (const auto& text : texts) + { + CAPTURE(text) + const json j = json::parse(text, nullptr, true, true, true); + const json_document d = json_document::parse(text, true, true, true); + check_exceptions(d.root(), j); + json_editable_document e = json_editable_document::parse(text, true, true, true); + check_exceptions(e.root(), j); + json m = e.root().materialize(); + check_edit_exceptions(e, e.root(), m); + } + } + + SECTION("generated documents") + { + json_view_test::generator gen; + for (int i = 0; i < 100; ++i) + { + std::string text; + gen.value(text, 0); + CAPTURE(text) + const json_document d = json_document::parse(text); + check_exceptions(d.root(), json::parse(text)); + } + } + + SECTION("edited documents: as those of materialize()") + { + const std::string text = R"({"a": [1, 2], "b": {"c": "\n", "d": true}, "e": [3, [4]], "f": "x"})"; + json_editable_document d = json_editable_document::parse(text); + d.set(d.root()["f"], 5); + d.push_back(d.root()["a"], json::parse(R"({"n": [null]})")); + d.erase(d.root()["e"], 0); + d.set(d.root()["b"], "c2", d.root()["e"]); + d.insert(d.root()["e"], 0, "new"); + json m = d.root().materialize(); + check_exceptions(d.root(), m); + check_edit_exceptions(d, d.root(), m); + // (a value inside a new container: a path, but no bytes) + std::string expected = "[json.exception.type_error.302] "; +#if JSON_DIAGNOSTICS + expected += "(/a/2/n/0) "; +#endif + expected += "type must be number, but is null"; + CHECK(exception_of_call([&] { static_cast(d.root()["a"][2]["n"][0].get()); }) == expected); + } + + SECTION("deep documents with siblings after every nested container") + { + // the search for the path pushes and pops one frame per level (the + // stack reallocates while it grows, and shrinks again at every + // sibling): run under a sanitizer, a reference to a popped or moved + // frame would show here + std::string text = "7"; + for (int level = 0; level < 40; ++level) + { + text = R"([[1, [2]], )" + text + R"(, {"k": [3, {"l": [4]}]}, [5, [6, [7]]]])"; + } + CAPTURE(text) + const json j = json::parse(text); + const json_document d = json_document::parse(text); + check_exceptions(d.root(), j); + json_editable_document e = json_editable_document::parse(text); + e.set(e.root()[0][0], 9); + check_exceptions(e.root(), e.root().materialize()); + } + + SECTION("examples") + { + const std::string text = R"({"people": [{"name": "a", "age": 1}, {"name": "c", "age": "x"}], "o": {"k": 1}})"; + const json_document d = json_document::parse(text); + std::string expected_age = "[json.exception.type_error.302] "; +#if JSON_DIAGNOSTICS + expected_age += "(/people/1/age) "; +#endif + expected_age += "(bytes 58-61) type must be number, but is string"; + CHECK(exception_of_call([&] { static_cast(d.root()["people"][1]["age"].get()); }) == expected_age); + + std::string expected_zz = "[json.exception.out_of_range.403] "; +#if JSON_DIAGNOSTICS + expected_zz += "(/people/0) "; +#endif + expected_zz += "(bytes 12-35) key 'zz' not found"; + CHECK(exception_of_call([&] { static_cast(d.root().at(json::json_pointer("/people/0/zz"))); }) == expected_zz); + + // the root has no path, and with JSON_DIAGNOSTICS then no bytes either (as basic_json) + std::string expected_missing = "[json.exception.out_of_range.403] "; +#if !JSON_DIAGNOSTICS + expected_missing += "(bytes 0-79) "; +#endif + expected_missing += "key 'missing' not found"; + CHECK(exception_of_call([&] { static_cast(d.root().at("missing")); }) == expected_missing); + } + + SECTION("duplicate keys: the value the view finds") + { + const json_document d = json_document::parse(R"({"x": "1", "x": "22"})"); + std::string expected = "[json.exception.type_error.302] "; +#if JSON_DIAGNOSTICS + expected += "(/x) "; +#endif + expected += "(bytes 6-9) type must be number, but is string"; + CHECK(exception_of_call([&] { static_cast(d.root()["x"].get()); }) == expected); + } +} +#endif From 67b103e5062b8fa443fd3a85d811d996bd39ae1d Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 1 Nov 2026 20:42:58 +0100 Subject: [PATCH 19/54] Fix clang-tidy errors in json_view tests on develop (#5989) readability-math-missing-parentheses in unit-json_view_builder.cpp (#5974) and performance-inefficient-string-concatenation in unit-json_view_positions.cpp. Signed-off-by: Niels Lohmann --- tests/src/unit-json_view_builder.cpp | 2 +- tests/src/unit-json_view_positions.cpp | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/src/unit-json_view_builder.cpp b/tests/src/unit-json_view_builder.cpp index 06059aba8..988c25771 100644 --- a/tests/src/unit-json_view_builder.cpp +++ b/tests/src/unit-json_view_builder.cpp @@ -692,7 +692,7 @@ TEST_CASE("json_view builder: fraction digits") check_same(number); check_same("[" + number + "]"); std::string pair; - pair.reserve(number.size() * 2 + 3); + pair.reserve((number.size() * 2) + 3); pair += '['; pair += number; pair += ','; diff --git a/tests/src/unit-json_view_positions.cpp b/tests/src/unit-json_view_positions.cpp index c6afd7fcb..966c4b46a 100644 --- a/tests/src/unit-json_view_positions.cpp +++ b/tests/src/unit-json_view_positions.cpp @@ -521,7 +521,8 @@ TEST_CASE("json_view exceptions name the path and the bytes of the value") std::string text = "7"; for (int level = 0; level < 40; ++level) { - text = R"([[1, [2]], )" + text + R"(, {"k": [3, {"l": [4]}]}, [5, [6, [7]]]])"; + text.insert(0, R"([[1, [2]], )"); + text += R"(, {"k": [3, {"l": [4]}]}, [5, [6, [7]]]])"; } CAPTURE(text) const json j = json::parse(text); From e2423a18574b18eda1425a38089766cb60663303 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 1 Nov 2026 20:43:07 +0100 Subject: [PATCH 20/54] CI: pin ubuntu-latest jobs to ubuntu-26.04 (#5983) GitHub is moving the ubuntu-latest label from Ubuntu 24.04 to 26.04 between October 19 and November 19, 2026 (actions/runner-images#14748). Pin every ubuntu-latest job to ubuntu-26.04 so the switch does not change the toolchain under our jobs, and so the behaviour does not depend on when GitHub reaches our runs. Jobs pinned to ubuntu-24.04 and ubuntu-24.04-arm are unchanged. Signed-off-by: Niels Lohmann --- .github/workflows/cancel_closed_pr_runs.yml | 2 +- .github/workflows/check_amalgamation.yml | 2 +- .github/workflows/check_docs_links.yml | 2 +- .../workflows/comment_check_amalgamation.yml | 2 +- .github/workflows/dependency-review.yml | 2 +- .github/workflows/flawfinder.yml | 2 +- .github/workflows/labeler.yml | 2 +- .github/workflows/publish_documentation.yml | 2 +- .github/workflows/scorecards.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/ubuntu.yml | 26 +++++++++---------- 11 files changed, 23 insertions(+), 23 deletions(-) diff --git a/.github/workflows/cancel_closed_pr_runs.yml b/.github/workflows/cancel_closed_pr_runs.yml index 93d65312d..91005de52 100644 --- a/.github/workflows/cancel_closed_pr_runs.yml +++ b/.github/workflows/cancel_closed_pr_runs.yml @@ -24,7 +24,7 @@ jobs: permissions: actions: write - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 steps: - name: Harden Runner diff --git a/.github/workflows/check_amalgamation.yml b/.github/workflows/check_amalgamation.yml index 66af126fa..253531e40 100644 --- a/.github/workflows/check_amalgamation.yml +++ b/.github/workflows/check_amalgamation.yml @@ -19,7 +19,7 @@ permissions: jobs: save: if: github.event_name == 'pull_request' - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 steps: - name: Harden Runner uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 diff --git a/.github/workflows/check_docs_links.yml b/.github/workflows/check_docs_links.yml index 12f5ad074..f7e28ab69 100644 --- a/.github/workflows/check_docs_links.yml +++ b/.github/workflows/check_docs_links.yml @@ -16,7 +16,7 @@ concurrency: jobs: check_docs_links: if: github.repository == 'nlohmann/json' - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 timeout-minutes: 30 steps: - name: Harden Runner diff --git a/.github/workflows/comment_check_amalgamation.yml b/.github/workflows/comment_check_amalgamation.yml index 04bfaae88..ba2fcad7b 100644 --- a/.github/workflows/comment_check_amalgamation.yml +++ b/.github/workflows/comment_check_amalgamation.yml @@ -12,7 +12,7 @@ jobs: comment: # push runs on develop have no PR to comment on (and no "pr" artifact) if: ${{ github.event.workflow_run.conclusion == 'failure' && github.event.workflow_run.event == 'pull_request' }} - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 permissions: contents: read actions: read diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index bd2d295a4..27c229572 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -18,7 +18,7 @@ permissions: jobs: dependency-review: - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 steps: - name: Harden Runner uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 diff --git a/.github/workflows/flawfinder.yml b/.github/workflows/flawfinder.yml index b3e2511ae..ffc8c2a43 100644 --- a/.github/workflows/flawfinder.yml +++ b/.github/workflows/flawfinder.yml @@ -24,7 +24,7 @@ on: jobs: flawfinder: name: Flawfinder - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 permissions: actions: read contents: read diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 35549b4e0..85c7a513d 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -19,7 +19,7 @@ jobs: contents: read pull-requests: write - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 steps: - name: Harden Runner diff --git a/.github/workflows/publish_documentation.yml b/.github/workflows/publish_documentation.yml index 12c55a0d4..ad53308d7 100644 --- a/.github/workflows/publish_documentation.yml +++ b/.github/workflows/publish_documentation.yml @@ -34,7 +34,7 @@ jobs: contents: write if: github.repository == 'nlohmann/json' - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 steps: - name: Harden Runner uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index 14b987200..62e1d504a 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -24,7 +24,7 @@ permissions: jobs: analysis: name: Scorecard analysis - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 permissions: # Needed to upload the results to code-scanning dashboard. security-events: write diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 41928ab20..da7d91fce 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -8,7 +8,7 @@ permissions: jobs: stale: - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 permissions: issues: write diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index a8cd05766..c93c0a706 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -18,7 +18,7 @@ concurrency: jobs: ci_test_gcc: - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 container: gcc:latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -130,7 +130,7 @@ jobs: run: cmake --build build --target ${{ matrix.target }} ci_static_analysis_clang: - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 container: silkeh/clang:22 strategy: matrix: @@ -149,7 +149,7 @@ jobs: run: cmake --build build --target ${{ matrix.target }} ci_cmake_options: - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 container: ubuntu:24.04 strategy: matrix: @@ -209,7 +209,7 @@ jobs: fail-on-error: false ci_test_compilers_gcc_old: - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 strategy: matrix: compiler: ['4.8', '4.9', '5', '6'] @@ -251,7 +251,7 @@ jobs: run: cmake --build build --target ci_test_compiler_g++-${{ matrix.compiler }} ci_test_compilers_gcc: - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 strategy: matrix: # older GCC docker images (4, 5, 6) fail to check out code @@ -269,7 +269,7 @@ jobs: run: cmake --build build --target ci_test_compiler_default ci_test_compilers_clang: - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 strategy: matrix: compiler: ['3.4', '3.5', '3.6', '3.7', '3.8', '3.9', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13', '14', '15-bullseye', '16', '17', '18', '19', '20', '21', '22', 'latest'] @@ -289,7 +289,7 @@ jobs: run: cmake --build build --target ci_test_compiler_default ci_test_standards_gcc: - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 container: gcc:latest strategy: matrix: @@ -306,7 +306,7 @@ jobs: run: cmake --build build --target ci_test_gcc_cxx${{ matrix.standard }} ci_test_standards_clang: - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 container: silkeh/clang:latest strategy: matrix: @@ -330,7 +330,7 @@ jobs: if: ${{ matrix.stdlib == 'libstdcxx' }} ci_cuda_example: - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 strategy: fail-fast: false matrix: @@ -356,7 +356,7 @@ jobs: strategy: matrix: container: ['gcc:latest', 'silkeh/clang:latest'] - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 container: ${{ matrix.container }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -383,7 +383,7 @@ jobs: run: cmake --build build --target ci_module_cpp20 ci_icpc: - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 # Intel discontinued the classic icc/icpc compiler in oneAPI 2024.0; this is # Intel's own last officially published image that still includes it. container: intel/oneapi-hpckit:2023.2.1-devel-ubuntu22.04 @@ -402,7 +402,7 @@ jobs: run: cmake --build build --target ci_icpc ci_icpx: - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 container: intel/oneapi-hpckit:latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -416,7 +416,7 @@ jobs: run: cmake --build build --target ci_icpx ci_nvhpc: - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 container: nvcr.io/nvidia/nvhpc:25.5-devel-cuda12.9-ubuntu22.04 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 From b2674a4a2fb6de549973208dc01c5c1864e2c1b9 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 1 Nov 2026 21:03:35 +0100 Subject: [PATCH 21/54] BSON reader: check array element names in place in strict mode (#5984) --- .../nlohmann/detail/input/binary_reader.hpp | 46 +++++++++++++++++-- single_include/nlohmann/json.hpp | 46 +++++++++++++++++-- tests/src/unit-bson.cpp | 38 +++++++++++++++ 3 files changed, 122 insertions(+), 8 deletions(-) diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index da5a0263e..8a6a0a6d5 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -467,11 +467,13 @@ class binary_reader const std::size_t element_type_parse_position = chars_read; // an array's elements are named "0", "1", ... in the wire format, - // and those names are not passed on: they are skipped unless - // error_handler_t::strict must reject ill-formed UTF-8 in them + // and those names are not passed on: they are skipped, and only + // checked if error_handler_t::strict must reject ill-formed UTF-8 const bool is_object = container_stack.back().is_object; key.clear(); - if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key, is_object || error_handler == error_handler_t::strict))) + if (JSON_HEDLEY_UNLIKELY(!(is_object || error_handler != error_handler_t::strict + ? get_bson_cstr(key, is_object) + : check_bson_cstr(key, std::integral_constant {})))) { return false; } @@ -495,7 +497,7 @@ class binary_reader @param[in] store whether the characters are needed; if not (the names of an array's elements), they are skipped and @a result may be left unchanged, and they are not checked against - @ref error_handler + @ref error_handler (@ref check_bson_cstr checks them) @return `true` if the \\x00-byte indicating the end of the string was encountered before the EOF; false` indicates an unexpected EOF. */ @@ -554,6 +556,42 @@ class binary_reader return true; } + /*! + @brief skip a C-style string that error_handler_t::strict must check + + The names of an array's elements are not passed on. On contiguous input, + a well-formed name is checked in place and skipped without being copied. + Otherwise (no \\x00-byte in the input that can be scanned, or ill-formed + UTF-8), it is read by @ref get_bson_cstr, which reports the error. + + @param[in,out] result an empty string, which may receive the name + @return whether the name was read and is well-formed + */ + bool check_bson_cstr(string_t& result, std::true_type /*bulk*/) + { + const std::size_t remaining = ia.bulk_remaining(); + if (remaining != 0) + { + const auto* const data = reinterpret_cast(ia.bulk_data()); + const std::size_t length = find_zero_byte(data, remaining); + if (length != remaining && JSON_HEDLEY_LIKELY(valid_utf8_string(data, length))) + { + // consume the string and its \x00-byte, as get_bson_cstr does + ia.bulk_skip(length + 1); + chars_read += length + 1; + current = 0x00; + return true; + } + } + return get_bson_cstr(result); + } + + /// input that is not contiguous: the name is read and checked byte by byte + bool check_bson_cstr(string_t& result, std::false_type /*bulk*/) + { + return get_bson_cstr(result); + } + /// input that is not contiguous: C-style strings are read byte by byte bool get_bson_cstr_bulk(string_t& /*result*/, const bool /*store*/, std::false_type /*bulk*/) const noexcept { diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 1fc214c5f..693320ed5 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -18474,11 +18474,13 @@ class binary_reader const std::size_t element_type_parse_position = chars_read; // an array's elements are named "0", "1", ... in the wire format, - // and those names are not passed on: they are skipped unless - // error_handler_t::strict must reject ill-formed UTF-8 in them + // and those names are not passed on: they are skipped, and only + // checked if error_handler_t::strict must reject ill-formed UTF-8 const bool is_object = container_stack.back().is_object; key.clear(); - if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key, is_object || error_handler == error_handler_t::strict))) + if (JSON_HEDLEY_UNLIKELY(!(is_object || error_handler != error_handler_t::strict + ? get_bson_cstr(key, is_object) + : check_bson_cstr(key, std::integral_constant {})))) { return false; } @@ -18502,7 +18504,7 @@ class binary_reader @param[in] store whether the characters are needed; if not (the names of an array's elements), they are skipped and @a result may be left unchanged, and they are not checked against - @ref error_handler + @ref error_handler (@ref check_bson_cstr checks them) @return `true` if the \\x00-byte indicating the end of the string was encountered before the EOF; false` indicates an unexpected EOF. */ @@ -18561,6 +18563,42 @@ class binary_reader return true; } + /*! + @brief skip a C-style string that error_handler_t::strict must check + + The names of an array's elements are not passed on. On contiguous input, + a well-formed name is checked in place and skipped without being copied. + Otherwise (no \\x00-byte in the input that can be scanned, or ill-formed + UTF-8), it is read by @ref get_bson_cstr, which reports the error. + + @param[in,out] result an empty string, which may receive the name + @return whether the name was read and is well-formed + */ + bool check_bson_cstr(string_t& result, std::true_type /*bulk*/) + { + const std::size_t remaining = ia.bulk_remaining(); + if (remaining != 0) + { + const auto* const data = reinterpret_cast(ia.bulk_data()); + const std::size_t length = find_zero_byte(data, remaining); + if (length != remaining && JSON_HEDLEY_LIKELY(valid_utf8_string(data, length))) + { + // consume the string and its \x00-byte, as get_bson_cstr does + ia.bulk_skip(length + 1); + chars_read += length + 1; + current = 0x00; + return true; + } + } + return get_bson_cstr(result); + } + + /// input that is not contiguous: the name is read and checked byte by byte + bool check_bson_cstr(string_t& result, std::false_type /*bulk*/) + { + return get_bson_cstr(result); + } + /// input that is not contiguous: C-style strings are read byte by byte bool get_bson_cstr_bulk(string_t& /*result*/, const bool /*store*/, std::false_type /*bulk*/) const noexcept { diff --git a/tests/src/unit-bson.cpp b/tests/src/unit-bson.cpp index f1f43ccd7..ac8e2b24f 100644 --- a/tests/src/unit-bson.cpp +++ b/tests/src/unit-bson.cpp @@ -23,6 +23,7 @@ using nlohmann::json; #include #include #include +#include #include #include #include "make_test_data_available.hpp" @@ -1199,6 +1200,31 @@ TEST_CASE("BSON keys from contiguous and stream input") from_stream = e.what(); } CHECK(from_vector == from_stream); + + // error_handler_t::strict checks the array element names in place on + // contiguous input and byte by byte otherwise; the names are + // well-formed, so both give the same value or error as above + std::string strict_from_vector; + std::string strict_from_list; + try + { + strict_from_vector = json::from_bson(input, true, true, json::error_handler_t::strict).dump(); + } + catch (const json::parse_error& e) + { + strict_from_vector = e.what(); + } + try + { + const std::list list(input.begin(), input.end()); + strict_from_list = json::from_bson(list.begin(), list.end(), true, true, json::error_handler_t::strict).dump(); + } + catch (const json::parse_error& e) + { + strict_from_list = e.what(); + } + CHECK(strict_from_vector == from_vector); + CHECK(strict_from_list == from_vector); } } @@ -1227,6 +1253,18 @@ TEST_CASE("BSON array element names") std::istringstream stream(std::string(reinterpret_cast(bson.data()), bson.size())); CHECK(json::from_bson(stream) == expected); + + // input that is not contiguous reads the names byte by byte and must reject + // the same name + const std::list list(bson.begin(), bson.end()); + CHECK_THROWS_WITH_AS(_ = json::from_bson(list.begin(), list.end(), true, true, json::error_handler_t::strict), "[json.exception.parse_error.113] parse error at byte 14: syntax error while parsing BSON key: invalid string: ill-formed UTF-8 byte", json::parse_error&); + + // without exceptions, the rejected name ends the parse + CHECK(json::from_bson(bson, true, false, json::error_handler_t::strict).is_discarded()); + + // well-formed names pass the check without being stored + const json valid = {{"a", {true, false, {{"k", json::array({1, 2})}}}}}; + CHECK(json::from_bson(json::to_bson(valid), true, true, json::error_handler_t::strict) == valid); } #endif From 76dbe8f9c96a7d5e46c7cf996cdc05d371419506 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Mon, 2 Nov 2026 17:47:21 +0100 Subject: [PATCH 22/54] Fix the red CI on develop: Visual Studio 2017 build of the BSON writer, Valgrind timeouts (#6009) * CI: give the Valgrind tests a timeout of one hour Under Valgrind, test-json_view_builder_cpp11 and test-ordered_json_cpp11 took 1435 s and 1390 s on develop (ddc34f5), close to ctest's default timeout of 1500 s. On the next run (b2674a4) the whole job was about 10 % slower and both timed out, failing ci_test_valgrind on develop and on every PR based on it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01N5NasAgbDGG28MqXjQvLqp Signed-off-by: Claude * BSON writer: drop the SFINAE guard that Visual Studio 2017 cannot deduce write_bson_document_single_pass() was a member template enabled by is_detected. MSVC 19.16 (Visual Studio 2017) fails to deduce it through the class-scope alias template (C2672, C2783) when it compiles to_bson() and to_bon8(), so every Visual Studio 2017 job on AppVeyor fails since #5979. The guard is not needed: as a non-template member of the class template, the function is only instantiated when called, and its only caller is the write_bson_document() overload for sinks that have written(), which already calls oa.written() and oa.rewind() itself. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01N5NasAgbDGG28MqXjQvLqp Signed-off-by: Claude --------- Signed-off-by: Claude Co-authored-by: Claude --- cmake/test.cmake | 4 +++- include/nlohmann/detail/output/binary_writer.hpp | 1 - single_include/nlohmann/json.hpp | 1 - 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/test.cmake b/cmake/test.cmake index 5922c5106..9a42ac263 100644 --- a/cmake/test.cmake +++ b/cmake/test.cmake @@ -178,8 +178,10 @@ function(_json_test_add_test test_name file main cxx_standard) COMMAND ${memcheck_command} $ ${DOCTEST_TEST_FILTER} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} ) + # under Valgrind the slowest tests take close to 25 minutes, so ctest's + # default timeout of 1500 seconds leaves no headroom set_tests_properties(${test_target}_valgrind PROPERTIES - LABELS "valgrind" FIXTURES_REQUIRED TEST_DATA + LABELS "valgrind" FIXTURES_REQUIRED TEST_DATA TIMEOUT 3600 ) endif() endfunction() diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 0c3c823ac..c9bdc696f 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -2422,7 +2422,6 @@ class binary_writer nothing large has been written then, and the caller writes the document in two passes, which reports that error */ - template::value, int>::type = 0> bool write_bson_document_single_pass(const BasicJsonType& document) { const std::size_t start = oa.written(); diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 693320ed5..2a7289203 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -30394,7 +30394,6 @@ class binary_writer nothing large has been written then, and the caller writes the document in two passes, which reports that error */ - template::value, int>::type = 0> bool write_bson_document_single_pass(const BasicJsonType& document) { const std::size_t start = oa.written(); From 03d408aafa4260f82b2340c88684c25698d36aac Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Mon, 2 Nov 2026 17:47:48 +0100 Subject: [PATCH 23/54] JSON_DIAGNOSTIC_POSITIONS: comparing a value-initialized iterator throws instead of crashing (#6003) * JSON_DIAGNOSTIC_POSITIONS: comparing a value-initialized iterator throws instead of crashing With JSON_DIAGNOSTIC_POSITIONS, comparing a value-initialized iterator (which has no container) with another iterator built the invalid_iterator.212 message from the byte positions of the iterator's value, which is a null pointer: UBSan reports "member call on null pointer" in exception::get_byte_positions(), then the process crashes. Every other configuration throws invalid_iterator.212. get_byte_positions() now returns no positions for a null value, as the JSON_DIAGNOSTICS path already does. Signed-off-by: Niels Lohmann * Declare the value-initialized iterator in the test const clang-tidy (misc-const-correctness) flags it, as it is only compared. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01N5NasAgbDGG28MqXjQvLqp Signed-off-by: Claude --------- Signed-off-by: Niels Lohmann Signed-off-by: Claude Co-authored-by: Claude --- include/nlohmann/detail/exceptions.hpp | 3 ++- single_include/nlohmann/json.hpp | 3 ++- tests/src/unit-diagnostic-positions.cpp | 11 +++++++++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/include/nlohmann/detail/exceptions.hpp b/include/nlohmann/detail/exceptions.hpp index 144bb770c..098d70495 100644 --- a/include/nlohmann/detail/exceptions.hpp +++ b/include/nlohmann/detail/exceptions.hpp @@ -222,7 +222,8 @@ class exception : public std::exception template static std::string get_byte_positions(const BasicJsonType* leaf_element) { - if ((leaf_element->start_pos() != std::string::npos) && (leaf_element->end_pos() != std::string::npos)) + // a value-initialized iterator has no value (invalid_iterator.212) + if (leaf_element != nullptr && leaf_element->start_pos() != std::string::npos && leaf_element->end_pos() != std::string::npos) { return concat("(bytes ", std::to_string(leaf_element->start_pos()), "-", std::to_string(leaf_element->end_pos()), ") "); } diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 2a7289203..958cc4ae4 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -5719,7 +5719,8 @@ class exception : public std::exception template static std::string get_byte_positions(const BasicJsonType* leaf_element) { - if ((leaf_element->start_pos() != std::string::npos) && (leaf_element->end_pos() != std::string::npos)) + // a value-initialized iterator has no value (invalid_iterator.212) + if (leaf_element != nullptr && leaf_element->start_pos() != std::string::npos && leaf_element->end_pos() != std::string::npos) { return concat("(bytes ", std::to_string(leaf_element->start_pos()), "-", std::to_string(leaf_element->end_pos()), ") "); } diff --git a/tests/src/unit-diagnostic-positions.cpp b/tests/src/unit-diagnostic-positions.cpp index 9e0625115..11f63fb59 100644 --- a/tests/src/unit-diagnostic-positions.cpp +++ b/tests/src/unit-diagnostic-positions.cpp @@ -243,3 +243,14 @@ TEST_CASE("values read from a binary format have no positions") check_no_positions(j); } } + +TEST_CASE("comparing a value-initialized iterator throws invalid_iterator.212") +{ + // such an iterator has no value to take the byte positions from; the + // comparison used to dereference the null pointer instead of throwing + json j = {1, 2}; + const json::iterator it{}; + CHECK_THROWS_WITH_AS(it == j.begin(), "[json.exception.invalid_iterator.212] cannot compare iterators of different containers", json::invalid_iterator); + CHECK_THROWS_WITH_AS(it < j.begin(), "[json.exception.invalid_iterator.212] cannot compare iterators of different containers", json::invalid_iterator); + CHECK_THROWS_WITH_AS(j.begin() == it, "[json.exception.invalid_iterator.212] cannot compare iterators of different containers", json::invalid_iterator); +} From 66e2eeeec06fbca6e00e7fa36f8af52d0e7575c9 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Mon, 2 Nov 2026 17:48:03 +0100 Subject: [PATCH 24/54] Do not construct the base class when destroying a container (#6000) * Do not construct the base class when destroying a container (#5995) json_value::destroy_container() and destroy_walk() used default- constructed basic_json values as stand-ins, which construct a CustomBaseClass that belongs to no value. A base class whose default constructor allocates made the noexcept destructor allocate again, and terminate if that allocation failed, which #5762 set out to prevent. Use data (type and value, no base class) as the stand-in instead, and pass the children's m_data to the walk helpers. Add a test with a counting base class: destroying a value constructs no base object and destroys exactly one per value, also for values nested deeper than the recursion depth limit. Signed-off-by: Niels Lohmann * Release the test value with = nullptr instead of reset() clang-tidy (readability-ambiguous-smartptr-reset-call) rejects j.reset() because the pointee inherits a reset() from counting_base. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01N5NasAgbDGG28MqXjQvLqp Signed-off-by: Claude * Use std::swap for the container stand-in in destroy_container The unqualified call after `using std::swap` would let a user-provided swap(json_value&, json_value&) intercept the destruction path. Call std::swap directly. Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann Signed-off-by: Claude Co-authored-by: Claude --- include/nlohmann/json.hpp | 131 ++++++++++++++------------- single_include/nlohmann/json.hpp | 131 ++++++++++++++------------- tests/src/unit-custom-base-class.cpp | 90 ++++++++++++++++++ 3 files changed, 224 insertions(+), 128 deletions(-) diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 7bed813df..8b8d3dbed 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -543,6 +543,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec //////////////////////// JSON_PRIVATE_UNLESS_TESTED: + // the type and value of a basic_json without its base class (defined + // below); the destruction helpers of json_value use it as a stand-in + struct data; + /*! @brief a JSON value @@ -719,27 +723,28 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec explicit json_value(array_t* value) noexcept : array(value) {} private: - // raw, allocation-free transfer of m_data from src to dst: no - // set_parents()/assert_invariant() (the former is O(#children) per - // call under JSON_DIAGNOSTICS, which would make the walk below - // quadratic); dst takes ownership, src is left as value_t::null. - static void take(basic_json& dst, basic_json& src) noexcept + // raw, allocation-free transfer of the type and value from src to + // dst: no set_parents()/assert_invariant() (the former is + // O(#children) per call under JSON_DIAGNOSTICS, which would make the + // walk below quadratic); dst takes ownership, src is left as + // value_t::null. + static void take(data& dst, data& src) noexcept { - dst.m_data.m_type = src.m_data.m_type; - dst.m_data.m_value = src.m_data.m_value; - src.m_data.m_type = value_t::null; + dst.m_type = src.m_type; + dst.m_value = src.m_value; + src.m_type = value_t::null; } // true if v is not an array/object, or is an already-empty one - static bool has_no_children(const basic_json& v) noexcept + static bool has_no_children(const data& v) noexcept { - if (v.m_data.m_type == value_t::array) + if (v.m_type == value_t::array) { - return v.m_data.m_value.array->empty(); + return v.m_value.array->empty(); } - if (v.m_data.m_type == value_t::object) + if (v.m_type == value_t::object) { - return v.m_data.m_value.object->empty(); + return v.m_value.object->empty(); } return true; } @@ -748,21 +753,21 @@ private: // its own, i.e. v can be freed in one go (its destructor then // destroys scalars and empty containers only, so it recurses at most // one level deep); stops at the first structured child it meets - static bool all_children_leaves(const basic_json& v) noexcept + static bool all_children_leaves(const data& v) noexcept { // array_t only needs forward iterators (see custom_array_type) - if (v.m_data.m_type == value_t::array) + if (v.m_type == value_t::array) { - const auto& a = *v.m_data.m_value.array; + const auto& a = *v.m_value.array; return std::all_of(a.begin(), a.end(), [](const basic_json & el) { - return has_no_children(el); + return has_no_children(el.m_data); }); } - const auto& o = *v.m_data.m_value.object; + const auto& o = *v.m_value.object; return std::all_of(o.begin(), o.end(), [](const typename object_t::value_type & el) { - return has_no_children(el.second); + return has_no_children(el.second.m_data); }); } @@ -794,30 +799,30 @@ private: // the child of a non-empty array/object v that the walk in // destroy_container() continues with (see walk_child_it() above) - static basic_json& walk_child(basic_json& v) + static data& walk_child(data& v) { - if (v.m_data.m_type == value_t::array) + if (v.m_type == value_t::array) { - return v.m_data.m_value.array->back(); + return v.m_value.array->back().m_data; } - JSON_ASSERT(v.m_data.m_type == value_t::object); - return walk_child_it(*v.m_data.m_value.object)->second; + JSON_ASSERT(v.m_type == value_t::object); + return walk_child_it(*v.m_value.object)->second.m_data; } // removes walk_child(v) from a non-empty array/object v; this never // allocates, and since it is only ever called when that child is a // scalar or an already-empty array/object, destroying it never // recurses more than one level deep (see destroy() below) - static void pop_walk_child(basic_json& v) + static void pop_walk_child(data& v) { - if (v.m_data.m_type == value_t::array) + if (v.m_type == value_t::array) { - v.m_data.m_value.array->pop_back(); + v.m_value.array->pop_back(); } else { - JSON_ASSERT(v.m_data.m_type == value_t::object); - v.m_data.m_value.object->erase(walk_child_it(*v.m_data.m_value.object)); + JSON_ASSERT(v.m_type == value_t::object); + v.m_value.object->erase(walk_child_it(*v.m_value.object)); } } @@ -826,23 +831,23 @@ private: // recurses at most one level deep); this is the same allocator-based // free the old recursive implementation used, just factored out so // every level of the walk in destroy() can share it - static void free_container(basic_json& v) noexcept + static void free_container(data& v) noexcept { JSON_ASSERT(has_no_children(v) || all_children_leaves(v)); - if (v.m_data.m_type == value_t::array) + if (v.m_type == value_t::array) { AllocatorType alloc; - std::allocator_traits::destroy(alloc, v.m_data.m_value.array); - std::allocator_traits::deallocate(alloc, v.m_data.m_value.array, 1); + std::allocator_traits::destroy(alloc, v.m_value.array); + std::allocator_traits::deallocate(alloc, v.m_value.array, 1); } else { - JSON_ASSERT(v.m_data.m_type == value_t::object); + JSON_ASSERT(v.m_type == value_t::object); AllocatorType alloc; - std::allocator_traits::destroy(alloc, v.m_data.m_value.object); - std::allocator_traits::deallocate(alloc, v.m_data.m_value.object, 1); + std::allocator_traits::destroy(alloc, v.m_value.object); + std::allocator_traits::deallocate(alloc, v.m_value.object, 1); } - v.m_data.m_type = value_t::null; // avoid a double free if v is later destructed + v.m_type = value_t::null; // avoid a double free if v is later destructed } public: @@ -873,14 +878,15 @@ public: // This json_value is not itself a basic_json, so the top-level // container is first moved into a local stand-in ("cur"); a - // default-constructed basic_json has a null pointer in its - // m_value (see data::m_value's initializer), so swapping it - // with *this leaves this union's own pointer null, and it is - // never looked at or freed a second time. - basic_json cur; - cur.m_data.m_type = t; - using std::swap; - swap(cur.m_data.m_value, *this); + // default-constructed data has a null pointer in its m_value + // (see data::m_value's initializer), so swapping it with *this + // leaves this union's own pointer null, and it is never looked + // at or freed a second time. The stand-ins are data, not + // basic_json, so that destroying a value never constructs a + // CustomBaseClass, whose constructor may allocate (#5995). + data cur; + cur.m_type = t; + std::swap(cur.m_value, *this); destroy_bounded(cur, 0); } @@ -889,7 +895,7 @@ public: // The first recursion_depth_limit() levels are destroyed by plain // recursion, which is the fastest way and cannot throw; anything // nested deeper is handed to destroy_walk(), which uses no call stack. - static void destroy_bounded(basic_json& v, std::size_t depth) noexcept + static void destroy_bounded(data& v, std::size_t depth) noexcept { if (JSON_HEDLEY_UNLIKELY(depth >= detail::recursion_depth_limit())) { @@ -897,23 +903,23 @@ public: return; } - if (v.m_data.m_type == value_t::array) + if (v.m_type == value_t::array) { - for (auto& child : *v.m_data.m_value.array) + for (auto& child : *v.m_value.array) { if (child.m_data.m_type == value_t::array || child.m_data.m_type == value_t::object) { - destroy_bounded(child, depth + 1); + destroy_bounded(child.m_data, depth + 1); } } } else { - for (auto& child : *v.m_data.m_value.object) + for (auto& child : *v.m_value.object) { if (child.second.m_data.m_type == value_t::array || child.second.m_data.m_type == value_t::object) { - destroy_bounded(child.second, depth + 1); + destroy_bounded(child.second.m_data, depth + 1); } } } @@ -940,18 +946,15 @@ public: // array/object, which neither allocates nor recurses more // than one level deep. // - // This json_value is not itself a basic_json, so the - // top-level container is first moved into a local stand-in - // ("cur"); a default-constructed basic_json has a null - // pointer in its m_value (see data::m_value's initializer), - // so swapping it with *this leaves this union's own pointer - // null, and it is never looked at or freed a second time. - static void destroy_walk(basic_json& top) noexcept + // Like destroy_container(), the walk keeps the containers it + // holds in data stand-ins (cur, prev, gp, tmp), which never + // construct a CustomBaseClass. + static void destroy_walk(data& top) noexcept { - basic_json cur; + data cur; take(cur, top); - basic_json prev; // value_t::null: no parent + data prev; // value_t::null: no parent // true right after cur was entered: its children have not been // looked at yet. If none of them has children of its own, cur is @@ -965,7 +968,7 @@ public: fresh = false; if (has_no_children(cur) || (fresh_now && all_children_leaves(cur))) { - if (prev.m_data.m_type == value_t::null) + if (prev.m_type == value_t::null) { free_container(cur); return; // back at the top with nothing left to do @@ -974,7 +977,7 @@ public: // ascend: detach the grandparent link from prev's // walk_child() slot, drop that (now null) slot, free cur // (it is empty), then move up one level - basic_json gp; + data gp; take(gp, walk_child(prev)); pop_walk_child(prev); @@ -985,7 +988,7 @@ public: continue; } - basic_json& cur_child_ref = walk_child(cur); + data& cur_child_ref = walk_child(cur); if (has_no_children(cur_child_ref)) { @@ -997,7 +1000,7 @@ public: // descend into the non-empty child, reversing the // link: its slot takes over prev, and the child becomes // the new cur - basic_json tmp; + data tmp; take(tmp, cur_child_ref); take(cur_child_ref, prev); take(prev, cur); diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 958cc4ae4..c5c3ec84f 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -38373,6 +38373,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec //////////////////////// JSON_PRIVATE_UNLESS_TESTED: + // the type and value of a basic_json without its base class (defined + // below); the destruction helpers of json_value use it as a stand-in + struct data; + /*! @brief a JSON value @@ -38549,27 +38553,28 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec explicit json_value(array_t* value) noexcept : array(value) {} private: - // raw, allocation-free transfer of m_data from src to dst: no - // set_parents()/assert_invariant() (the former is O(#children) per - // call under JSON_DIAGNOSTICS, which would make the walk below - // quadratic); dst takes ownership, src is left as value_t::null. - static void take(basic_json& dst, basic_json& src) noexcept + // raw, allocation-free transfer of the type and value from src to + // dst: no set_parents()/assert_invariant() (the former is + // O(#children) per call under JSON_DIAGNOSTICS, which would make the + // walk below quadratic); dst takes ownership, src is left as + // value_t::null. + static void take(data& dst, data& src) noexcept { - dst.m_data.m_type = src.m_data.m_type; - dst.m_data.m_value = src.m_data.m_value; - src.m_data.m_type = value_t::null; + dst.m_type = src.m_type; + dst.m_value = src.m_value; + src.m_type = value_t::null; } // true if v is not an array/object, or is an already-empty one - static bool has_no_children(const basic_json& v) noexcept + static bool has_no_children(const data& v) noexcept { - if (v.m_data.m_type == value_t::array) + if (v.m_type == value_t::array) { - return v.m_data.m_value.array->empty(); + return v.m_value.array->empty(); } - if (v.m_data.m_type == value_t::object) + if (v.m_type == value_t::object) { - return v.m_data.m_value.object->empty(); + return v.m_value.object->empty(); } return true; } @@ -38578,21 +38583,21 @@ private: // its own, i.e. v can be freed in one go (its destructor then // destroys scalars and empty containers only, so it recurses at most // one level deep); stops at the first structured child it meets - static bool all_children_leaves(const basic_json& v) noexcept + static bool all_children_leaves(const data& v) noexcept { // array_t only needs forward iterators (see custom_array_type) - if (v.m_data.m_type == value_t::array) + if (v.m_type == value_t::array) { - const auto& a = *v.m_data.m_value.array; + const auto& a = *v.m_value.array; return std::all_of(a.begin(), a.end(), [](const basic_json & el) { - return has_no_children(el); + return has_no_children(el.m_data); }); } - const auto& o = *v.m_data.m_value.object; + const auto& o = *v.m_value.object; return std::all_of(o.begin(), o.end(), [](const typename object_t::value_type & el) { - return has_no_children(el.second); + return has_no_children(el.second.m_data); }); } @@ -38624,30 +38629,30 @@ private: // the child of a non-empty array/object v that the walk in // destroy_container() continues with (see walk_child_it() above) - static basic_json& walk_child(basic_json& v) + static data& walk_child(data& v) { - if (v.m_data.m_type == value_t::array) + if (v.m_type == value_t::array) { - return v.m_data.m_value.array->back(); + return v.m_value.array->back().m_data; } - JSON_ASSERT(v.m_data.m_type == value_t::object); - return walk_child_it(*v.m_data.m_value.object)->second; + JSON_ASSERT(v.m_type == value_t::object); + return walk_child_it(*v.m_value.object)->second.m_data; } // removes walk_child(v) from a non-empty array/object v; this never // allocates, and since it is only ever called when that child is a // scalar or an already-empty array/object, destroying it never // recurses more than one level deep (see destroy() below) - static void pop_walk_child(basic_json& v) + static void pop_walk_child(data& v) { - if (v.m_data.m_type == value_t::array) + if (v.m_type == value_t::array) { - v.m_data.m_value.array->pop_back(); + v.m_value.array->pop_back(); } else { - JSON_ASSERT(v.m_data.m_type == value_t::object); - v.m_data.m_value.object->erase(walk_child_it(*v.m_data.m_value.object)); + JSON_ASSERT(v.m_type == value_t::object); + v.m_value.object->erase(walk_child_it(*v.m_value.object)); } } @@ -38656,23 +38661,23 @@ private: // recurses at most one level deep); this is the same allocator-based // free the old recursive implementation used, just factored out so // every level of the walk in destroy() can share it - static void free_container(basic_json& v) noexcept + static void free_container(data& v) noexcept { JSON_ASSERT(has_no_children(v) || all_children_leaves(v)); - if (v.m_data.m_type == value_t::array) + if (v.m_type == value_t::array) { AllocatorType alloc; - std::allocator_traits::destroy(alloc, v.m_data.m_value.array); - std::allocator_traits::deallocate(alloc, v.m_data.m_value.array, 1); + std::allocator_traits::destroy(alloc, v.m_value.array); + std::allocator_traits::deallocate(alloc, v.m_value.array, 1); } else { - JSON_ASSERT(v.m_data.m_type == value_t::object); + JSON_ASSERT(v.m_type == value_t::object); AllocatorType alloc; - std::allocator_traits::destroy(alloc, v.m_data.m_value.object); - std::allocator_traits::deallocate(alloc, v.m_data.m_value.object, 1); + std::allocator_traits::destroy(alloc, v.m_value.object); + std::allocator_traits::deallocate(alloc, v.m_value.object, 1); } - v.m_data.m_type = value_t::null; // avoid a double free if v is later destructed + v.m_type = value_t::null; // avoid a double free if v is later destructed } public: @@ -38703,14 +38708,15 @@ public: // This json_value is not itself a basic_json, so the top-level // container is first moved into a local stand-in ("cur"); a - // default-constructed basic_json has a null pointer in its - // m_value (see data::m_value's initializer), so swapping it - // with *this leaves this union's own pointer null, and it is - // never looked at or freed a second time. - basic_json cur; - cur.m_data.m_type = t; - using std::swap; - swap(cur.m_data.m_value, *this); + // default-constructed data has a null pointer in its m_value + // (see data::m_value's initializer), so swapping it with *this + // leaves this union's own pointer null, and it is never looked + // at or freed a second time. The stand-ins are data, not + // basic_json, so that destroying a value never constructs a + // CustomBaseClass, whose constructor may allocate (#5995). + data cur; + cur.m_type = t; + std::swap(cur.m_value, *this); destroy_bounded(cur, 0); } @@ -38719,7 +38725,7 @@ public: // The first recursion_depth_limit() levels are destroyed by plain // recursion, which is the fastest way and cannot throw; anything // nested deeper is handed to destroy_walk(), which uses no call stack. - static void destroy_bounded(basic_json& v, std::size_t depth) noexcept + static void destroy_bounded(data& v, std::size_t depth) noexcept { if (JSON_HEDLEY_UNLIKELY(depth >= detail::recursion_depth_limit())) { @@ -38727,23 +38733,23 @@ public: return; } - if (v.m_data.m_type == value_t::array) + if (v.m_type == value_t::array) { - for (auto& child : *v.m_data.m_value.array) + for (auto& child : *v.m_value.array) { if (child.m_data.m_type == value_t::array || child.m_data.m_type == value_t::object) { - destroy_bounded(child, depth + 1); + destroy_bounded(child.m_data, depth + 1); } } } else { - for (auto& child : *v.m_data.m_value.object) + for (auto& child : *v.m_value.object) { if (child.second.m_data.m_type == value_t::array || child.second.m_data.m_type == value_t::object) { - destroy_bounded(child.second, depth + 1); + destroy_bounded(child.second.m_data, depth + 1); } } } @@ -38770,18 +38776,15 @@ public: // array/object, which neither allocates nor recurses more // than one level deep. // - // This json_value is not itself a basic_json, so the - // top-level container is first moved into a local stand-in - // ("cur"); a default-constructed basic_json has a null - // pointer in its m_value (see data::m_value's initializer), - // so swapping it with *this leaves this union's own pointer - // null, and it is never looked at or freed a second time. - static void destroy_walk(basic_json& top) noexcept + // Like destroy_container(), the walk keeps the containers it + // holds in data stand-ins (cur, prev, gp, tmp), which never + // construct a CustomBaseClass. + static void destroy_walk(data& top) noexcept { - basic_json cur; + data cur; take(cur, top); - basic_json prev; // value_t::null: no parent + data prev; // value_t::null: no parent // true right after cur was entered: its children have not been // looked at yet. If none of them has children of its own, cur is @@ -38795,7 +38798,7 @@ public: fresh = false; if (has_no_children(cur) || (fresh_now && all_children_leaves(cur))) { - if (prev.m_data.m_type == value_t::null) + if (prev.m_type == value_t::null) { free_container(cur); return; // back at the top with nothing left to do @@ -38804,7 +38807,7 @@ public: // ascend: detach the grandparent link from prev's // walk_child() slot, drop that (now null) slot, free cur // (it is empty), then move up one level - basic_json gp; + data gp; take(gp, walk_child(prev)); pop_walk_child(prev); @@ -38815,7 +38818,7 @@ public: continue; } - basic_json& cur_child_ref = walk_child(cur); + data& cur_child_ref = walk_child(cur); if (has_no_children(cur_child_ref)) { @@ -38827,7 +38830,7 @@ public: // descend into the non-empty child, reversing the // link: its slot takes over prev, and the child becomes // the new cur - basic_json tmp; + data tmp; take(tmp, cur_child_ref); take(cur_child_ref, prev); take(prev, cur); diff --git a/tests/src/unit-custom-base-class.cpp b/tests/src/unit-custom-base-class.cpp index 673c7ddbb..71698d041 100644 --- a/tests/src/unit-custom-base-class.cpp +++ b/tests/src/unit-custom-base-class.cpp @@ -7,6 +7,7 @@ // SPDX-License-Identifier: MIT #include +#include #include #include #include @@ -495,3 +496,92 @@ TEST_CASE("Regression test for issue #5674 - copy construction must not require CHECK(*c == 1); } } + +// counts the constructions and destructions of the base class (#5995) +struct counting_base +{ + static std::size_t constructions; + static std::size_t destructions; + + counting_base() noexcept + { + ++constructions; + } + counting_base(const counting_base& /*unused*/) noexcept + { + ++constructions; + } + counting_base(counting_base&& /*unused*/) noexcept + { + ++constructions; + } + counting_base& operator=(const counting_base& /*unused*/) noexcept = default; + counting_base& operator=(counting_base&& /*unused*/) noexcept = default; + ~counting_base() + { + ++destructions; + } + + static void reset() noexcept + { + constructions = 0; + destructions = 0; + } +}; + +std::size_t counting_base::constructions = 0; +std::size_t counting_base::destructions = 0; + +using json_with_counting_base = nlohmann::json::with_base_class_t; + +TEST_CASE("destroying a value does not construct the custom base class") +{ + // regression test for #5995: destroying an array or object used a + // default-constructed basic_json as a stand-in, which constructed a base + // class that belonged to no value (and could allocate in the noexcept + // destructor); now every destroyed value destroys exactly its own base + // object, and none is constructed + const auto check_destroy = [](std::unique_ptr j, std::size_t values) + { + counting_base::reset(); + j = nullptr; + CHECK(counting_base::constructions == 0); + CHECK(counting_base::destructions == values); + }; + + SECTION("scalar") + { + check_destroy(std::unique_ptr(new json_with_counting_base(42)), 1); + } + + SECTION("array and object") + { + check_destroy(std::unique_ptr(new json_with_counting_base(json_with_counting_base::parse(R"({"a":[1,2,3],"b":{"c":true}})"))), 7); + check_destroy(std::unique_ptr(new json_with_counting_base(json_with_counting_base::array())), 1); + } + + SECTION("nested deeper than the recursion depth limit") + { + // alternating arrays and objects, so both kinds of container are + // walked by destroy_walk() + constexpr std::size_t depth = 1000; + std::unique_ptr j(new json_with_counting_base(json_with_counting_base::array({1, 2}))); + for (std::size_t i = 0; i < depth; ++i) + { + json_with_counting_base outer = (i % 2 == 0) ? json_with_counting_base::object() : json_with_counting_base::array(); + if (outer.is_object()) + { + outer["x"] = std::move(*j); + outer["y"] = nullptr; + } + else + { + outer.push_back(std::move(*j)); + outer.push_back(true); + } + *j = std::move(outer); + } + // the innermost array with its two elements, and two values per level + check_destroy(std::move(j), 3 + (2 * depth)); + } +} From bdc737db6d1e973247f9b2d3e2e1d04e11de3e63 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Mon, 2 Nov 2026 17:48:16 +0100 Subject: [PATCH 25/54] json_view: dump() without StringType::capacity() (#5999) * json_view: dump() without StringType::capacity() (#5994) output_buffer::grow() compared the output size with m_out.capacity(), which the StringType requirements do not include, so dump() did not compile with the tests' alt_string. Keep what was last reserved in the buffer itself and use it as a lower bound of the capacity instead. Add json_view tests with alt_string (read-only and editable documents, dump with several growth steps, save/load), and document reserve(n) and resize(n) as the StringType requirements of basic_json_view::dump. Signed-off-by: Niels Lohmann * Use a raw string literal in the alt-string json_view test clang-tidy (modernize-raw-string-literal) rejects the escaped literal. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01N5NasAgbDGG28MqXjQvLqp Signed-off-by: Claude --------- Signed-off-by: Niels Lohmann Signed-off-by: Claude Co-authored-by: Claude --- .../features/types/template_parameters.md | 1 + include/nlohmann/detail/view/serializer.hpp | 13 +++-- single_include/nlohmann/json_view.hpp | 13 +++-- tests/src/unit-alt-string.cpp | 53 +++++++++++++++++++ 4 files changed, 72 insertions(+), 8 deletions(-) diff --git a/docs/mkdocs/docs/features/types/template_parameters.md b/docs/mkdocs/docs/features/types/template_parameters.md index f3868f594..6ae6820f5 100644 --- a/docs/mkdocs/docs/features/types/template_parameters.md +++ b/docs/mkdocs/docs/features/types/template_parameters.md @@ -406,6 +406,7 @@ using array_t = ArrayType>; | `#!cpp operator<<(std::ostream&, const json_pointer&)` | streamability to `#!cpp std::ostream` | | [`to_string`](../../api/basic_json/to_string.md) | conversion of `StringType` to `#!cpp std::string` (the function returns a `#!cpp std::string`) | | exception messages | `data()` and `size()`, or `begin()` and `end()` | +| [`basic_json_view::dump`](../../api/basic_json_view/dump.md) | `reserve(n)` and `resize(n)` (one argument); `capacity()` is **not** required | ### Compatible types diff --git a/include/nlohmann/detail/view/serializer.hpp b/include/nlohmann/detail/view/serializer.hpp index 2bfa2d2c0..28e1d647a 100644 --- a/include/nlohmann/detail/view/serializer.hpp +++ b/include/nlohmann/detail/view/serializer.hpp @@ -47,6 +47,7 @@ class output_buffer : m_out(sized(out, estimate)) , m_pos(&m_out[0]) , m_end(m_pos + m_out.size()) + , m_reserved((std::max)(estimate, m_out.size())) {} void finish() @@ -116,11 +117,12 @@ class output_buffer const auto used = static_cast(m_pos - m_out.data()); // (a step does not go beyond the reserved estimate, so that a good // estimate is never copied to a larger allocation; not a clamp: the - // lower bound used + n + 256 can exceed capacity(), and then it must win) - const std::size_t size = (std::max)((std::min)(m_out.size() + step, m_out.capacity()), used + n + 256); - if (size > m_out.capacity()) + // lower bound used + n + 256 can exceed m_reserved, and then it must win) + const std::size_t size = (std::max)((std::min)(m_out.size() + step, m_reserved), used + n + 256); + if (size > m_reserved) { - m_out.reserve((std::max)(m_out.capacity() * 2, size)); + m_reserved = (std::max)(m_reserved * 2, size); + m_out.reserve(m_reserved); } m_out.resize(size); m_pos = &m_out[0] + used; @@ -130,6 +132,9 @@ class output_buffer StringType& m_out; char* m_pos; char* m_end; + /// what was last passed to reserve(), a lower bound of the capacity + /// (StringType need not have a capacity() member) + std::size_t m_reserved; }; /// The length of the run at s that dump() writes unchanged without diff --git a/single_include/nlohmann/json_view.hpp b/single_include/nlohmann/json_view.hpp index e24f125bf..6bb6ec771 100644 --- a/single_include/nlohmann/json_view.hpp +++ b/single_include/nlohmann/json_view.hpp @@ -8511,6 +8511,7 @@ class output_buffer : m_out(sized(out, estimate)) , m_pos(&m_out[0]) , m_end(m_pos + m_out.size()) + , m_reserved((std::max)(estimate, m_out.size())) {} void finish() @@ -8580,11 +8581,12 @@ class output_buffer const auto used = static_cast(m_pos - m_out.data()); // (a step does not go beyond the reserved estimate, so that a good // estimate is never copied to a larger allocation; not a clamp: the - // lower bound used + n + 256 can exceed capacity(), and then it must win) - const std::size_t size = (std::max)((std::min)(m_out.size() + step, m_out.capacity()), used + n + 256); - if (size > m_out.capacity()) + // lower bound used + n + 256 can exceed m_reserved, and then it must win) + const std::size_t size = (std::max)((std::min)(m_out.size() + step, m_reserved), used + n + 256); + if (size > m_reserved) { - m_out.reserve((std::max)(m_out.capacity() * 2, size)); + m_reserved = (std::max)(m_reserved * 2, size); + m_out.reserve(m_reserved); } m_out.resize(size); m_pos = &m_out[0] + used; @@ -8594,6 +8596,9 @@ class output_buffer StringType& m_out; char* m_pos; char* m_end; + /// what was last passed to reserve(), a lower bound of the capacity + /// (StringType need not have a capacity() member) + std::size_t m_reserved; }; /// The length of the run at s that dump() writes unchanged without diff --git a/tests/src/unit-alt-string.cpp b/tests/src/unit-alt-string.cpp index 75feb74e8..4a0260a10 100644 --- a/tests/src/unit-alt-string.cpp +++ b/tests/src/unit-alt-string.cpp @@ -10,6 +10,7 @@ #include "doctest_compatibility.h" #include +#include #include #include @@ -483,4 +484,56 @@ TEST_CASE("alternative string type") } } +TEST_CASE("alternative string type with json_view") +{ + // regression test for #5994: basic_json_view::dump() called + // StringType::capacity(), which alt_string (and the documented StringType + // requirements) do not provide + using alt_document = nlohmann::basic_json_document; + using alt_editable_document = nlohmann::basic_json_document; + + const char* text = R"({"a":[1,"x\u00e4"],"b":{"c":true,"d":null},"e":2.5})"; + const alt_json j = alt_json::parse(text); + + SECTION("read-only document") + { + const alt_document d = alt_document::parse(text); + CHECK(d.root().dump() == j.dump()); + CHECK(d.root().dump(2) == j.dump(2)); + CHECK(d.root().dump(-1, ' ', true) == j.dump(-1, ' ', true)); + CHECK(d.root().materialize() == j); + CHECK(d.root().contains(alt_string("b"))); + CHECK(d.root()[alt_string("b")][alt_string("c")].dump() == "true"); + } + + SECTION("long output") + { + // more than one growth step of the output buffer + alt_json big = alt_json::array(); + for (int i = 0; i < 20000; ++i) + { + big.push_back(alt_json::parse(text)); + } + const alt_string big_text = big.dump(); + const alt_document d = alt_document::parse(big_text); + CHECK(d.root().dump() == big_text); + CHECK(d.root().dump(4) == big.dump(4)); + } + + SECTION("editable document") + { + alt_editable_document d = alt_editable_document::parse(text); + d.set(d.root(), "f", "new"); + d.push_back(d.root()[alt_string("a")], 3); + alt_json expected = j; + expected["f"] = "new"; + expected["a"].push_back(3); + CHECK(d.root().materialize() == expected); + CHECK(alt_json::parse(d.root().dump()) == expected); + + const alt_editable_document loaded = alt_editable_document::load(d.save()); + CHECK(loaded.root().materialize() == expected); + } +} + DOCTEST_CLANG_SUPPRESS_WARNING_POP From 494a74c2427e496d586c3f4af9c9d9fca49b342a Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Mon, 2 Nov 2026 17:48:29 +0100 Subject: [PATCH 26/54] insert(): strong guarantee for single elements, basic guarantee for the rest (#5996) * Make basic_json::insert() provide its documented strong exception guarantee (#5986) A failed allocation while the new elements were copied left the value changed, and with JSON_DIAGNOSTICS stale parent pointers behind. The extra work now happens almost only on the failure path: - Array insert(pos, cnt, val) and insert(pos, first, last) append the copies at the end and rotate them into place. If a copy throws, the appended elements are erased again (noexcept) and the parents are set. Moving a basic_json cannot throw, so reallocation is safe. This is no slower than vector::insert (faster for numbers), needs no temporary array, and keeps the elements that libstdc++ lost on reallocation. - Array insert(pos, val) copies a non-scalar value first and inserts it by move, except at the end, where vector::insert is strong. Scalars keep the direct path. - The object range insert(first, last) into an empty object uses the container's own range insertion and clears the object on an exception. Into a non-empty object it inserts member by member and logs one bit per source member (a new member is detected by the size change; no allocation for the first 256 members); on an exception the members added are erased by walking the source range. No assumption that key_type is string_t. - The parents are set when a failed insertion is undone. The regression test fails every allocation of every insert() overload in turn, for json and ordered_json, and checks the value and its parents. Signed-off-by: Niels Lohmann * Fix clang-tidy and -Wnrvo findings in insert rollback helpers Declare the rollback helpers non-movable, replace the nested conditional and C array in object_insert_rollback, silence the intentional copy in the allocator test, and avoid the missed NRVO in the test factory lambda. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01N5NasAgbDGG28MqXjQvLqp Signed-off-by: Claude * insert(): strong guarantee for single elements, basic for the rest Rework of the previous two commits, following the discussion in #5996: the standard containers give the strong guarantee only for a single element (std::vector: at the end, or if the exception does not come from copying T; std::map: single-element insert), and copying a basic_json allocates. Rebuilding a rollback for the count, range and object-range overloads on top of array_t/object_t made the code verbose and slower. - insert(pos, const basic_json&): at end() forward as before (the standard covers it); elsewhere copy first and insert by move, as libstdc++ does internally (libc++ shifts the tail before copying). - insert_iterator() and object insert(first, last) set the parents in a scope guard, so a failed insertion leaves a valid value with JSON_DIAGNOSTICS too (the next assert_invariant() aborted before). - Docs: strong guarantee for (1), basic guarantee for (2)-(5); the argument checks throw before anything is changed. - Test: allocation-failure sweep checks the strong guarantee for (1) and validity, type and parents for the other overloads. Signed-off-by: Niels Lohmann * insert(): document the exception safety of overloads 2 to 5 once The four basic-guarantee statements were identical up to array/object. Signed-off-by: Niels Lohmann * insert(): simplify the wording of the basic guarantee Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann Signed-off-by: Claude Co-authored-by: Claude --- docs/mkdocs/docs/api/basic_json/insert.md | 7 +- include/nlohmann/json.hpp | 40 +++- single_include/nlohmann/json.hpp | 40 +++- tests/src/unit-allocator.cpp | 220 ++++++++++++++++++++++ 4 files changed, 298 insertions(+), 9 deletions(-) diff --git a/docs/mkdocs/docs/api/basic_json/insert.md b/docs/mkdocs/docs/api/basic_json/insert.md index 12619a2d8..721cb8ae7 100644 --- a/docs/mkdocs/docs/api/basic_json/insert.md +++ b/docs/mkdocs/docs/api/basic_json/insert.md @@ -65,7 +65,12 @@ the same index, which is now a different value. ## Exception safety -Strong exception safety: if an exception occurs, the original value stays intact. +Overload (1) gives the strong guarantee: if an exception is thrown, there are no changes in the JSON value. + +Overloads (2) to (5) give the basic guarantee: if an exception is thrown while the elements or members are copied +(for example, `std::bad_alloc`), the JSON value may be partially modified, but it stays a valid array or object. + +The exceptions listed below are thrown before the JSON value is changed. ## Exceptions diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 8b8d3dbed..28cad76ba 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -4917,6 +4917,30 @@ public: return {it, res.second}; } + private: + /// Sets the parents of a value when it goes out of scope, also when an + /// exception is thrown: a container insertion that throws may already + /// have added elements, whose parents must be set for the value to stay + /// valid (basic exception guarantee). Does nothing without JSON_DIAGNOSTICS. + class set_parents_on_exit + { + public: + explicit set_parents_on_exit(basic_json* v) noexcept : m_value(v) {} + set_parents_on_exit(const set_parents_on_exit&) = delete; + set_parents_on_exit& operator=(const set_parents_on_exit&) = delete; + set_parents_on_exit(set_parents_on_exit&&) = delete; + set_parents_on_exit& operator=(set_parents_on_exit&&) = delete; + + ~set_parents_on_exit() + { + m_value->set_parents(); + } + + private: + basic_json* m_value; + }; + + public: /// Helper for insertion of an iterator /// @note: This uses std::distance to support GCC 4.8, /// see https://github.com/nlohmann/json/pull/1257 @@ -4926,6 +4950,7 @@ public: iterator result(this); JSON_ASSERT(m_data.m_value.array != nullptr); + const set_parents_on_exit parents(this); auto insert_pos = std::distance(m_data.m_value.array->begin(), pos.m_it.array_iterator); m_data.m_value.array->insert(pos.m_it.array_iterator, std::forward(args)...); result.m_it.array_iterator = m_data.m_value.array->begin() + insert_pos; @@ -4934,7 +4959,6 @@ public: // result.m_it.array_iterator = m_data.m_value.array->insert(pos.m_it.array_iterator, cnt, val); // but the return value of insert is missing in GCC 4.8, so it is written this way instead. - set_parents(); return result; } @@ -4954,8 +4978,16 @@ public: throw_iterator_from_other_value(this); } - // insert to array and return iterator - return insert_iterator(pos, val); + // vector::insert has no effects if copying a single element to the + // end throws; elsewhere, it may already have shifted the elements, so + // copy first and insert by move, which cannot throw but by allocating + if (pos.m_it.array_iterator == m_data.m_value.array->end()) + { + return insert_iterator(pos, val); + } + + basic_json tmp(val); + return insert_iterator(pos, std::move(tmp)); } /// @brief inserts element into array @@ -5087,8 +5119,8 @@ public: detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterators first and last must point to objects", this); } + const set_parents_on_exit parents(this); m_data.m_value.object->insert(first.m_it.object_iterator, last.m_it.object_iterator); - set_parents(); } /// @brief updates a JSON object from another object, overwriting existing keys diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index c5c3ec84f..80f938972 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -42747,6 +42747,30 @@ public: return {it, res.second}; } + private: + /// Sets the parents of a value when it goes out of scope, also when an + /// exception is thrown: a container insertion that throws may already + /// have added elements, whose parents must be set for the value to stay + /// valid (basic exception guarantee). Does nothing without JSON_DIAGNOSTICS. + class set_parents_on_exit + { + public: + explicit set_parents_on_exit(basic_json* v) noexcept : m_value(v) {} + set_parents_on_exit(const set_parents_on_exit&) = delete; + set_parents_on_exit& operator=(const set_parents_on_exit&) = delete; + set_parents_on_exit(set_parents_on_exit&&) = delete; + set_parents_on_exit& operator=(set_parents_on_exit&&) = delete; + + ~set_parents_on_exit() + { + m_value->set_parents(); + } + + private: + basic_json* m_value; + }; + + public: /// Helper for insertion of an iterator /// @note: This uses std::distance to support GCC 4.8, /// see https://github.com/nlohmann/json/pull/1257 @@ -42756,6 +42780,7 @@ public: iterator result(this); JSON_ASSERT(m_data.m_value.array != nullptr); + const set_parents_on_exit parents(this); auto insert_pos = std::distance(m_data.m_value.array->begin(), pos.m_it.array_iterator); m_data.m_value.array->insert(pos.m_it.array_iterator, std::forward(args)...); result.m_it.array_iterator = m_data.m_value.array->begin() + insert_pos; @@ -42764,7 +42789,6 @@ public: // result.m_it.array_iterator = m_data.m_value.array->insert(pos.m_it.array_iterator, cnt, val); // but the return value of insert is missing in GCC 4.8, so it is written this way instead. - set_parents(); return result; } @@ -42784,8 +42808,16 @@ public: throw_iterator_from_other_value(this); } - // insert to array and return iterator - return insert_iterator(pos, val); + // vector::insert has no effects if copying a single element to the + // end throws; elsewhere, it may already have shifted the elements, so + // copy first and insert by move, which cannot throw but by allocating + if (pos.m_it.array_iterator == m_data.m_value.array->end()) + { + return insert_iterator(pos, val); + } + + basic_json tmp(val); + return insert_iterator(pos, std::move(tmp)); } /// @brief inserts element into array @@ -42917,8 +42949,8 @@ public: detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterators first and last must point to objects", this); } + const set_parents_on_exit parents(this); m_data.m_value.object->insert(first.m_it.object_iterator, last.m_it.object_iterator); - set_parents(); } /// @brief updates a JSON object from another object, overwriting existing keys diff --git a/tests/src/unit-allocator.cpp b/tests/src/unit-allocator.cpp index 4062870d5..02b145aae 100644 --- a/tests/src/unit-allocator.cpp +++ b/tests/src/unit-allocator.cpp @@ -12,6 +12,7 @@ #include using nlohmann::json; +#include #include #if JSON_HAS_RANGES #include @@ -804,3 +805,222 @@ TEST_CASE("a failed allocation leaves the value unchanged") #endif } #endif + +// the no-exceptions CI job has no std::bad_alloc to catch; with iterator +// debugging (MSVC STL debug builds) containers allocate inside noexcept +// functions, so failing an allocation there terminates the program +#if !defined(JSON_NOEXCEPTION) && !(defined(_ITERATOR_DEBUG_LEVEL) && _ITERATOR_DEBUG_LEVEL > 0) +namespace +{ +// whether every child of j names j as its parent (always true without +// JSON_DIAGNOSTICS, where no parents are stored) +template +bool children_name_their_parent(const BasicJsonType& j) +{ +#if JSON_DIAGNOSTICS + if (j.is_array()) + { + for (const auto& element : *j.m_data.m_value.array) + { + if (element.m_parent != &j) + { + return false; + } + } + } + else if (j.is_object()) + { + for (const auto& element : *j.m_data.m_value.object) + { + if (element.second.m_parent != &j) + { + return false; + } + } + } +#else + static_cast(j); +#endif + return true; +} + +enum class guarantee +{ + basic, // the value stays valid and keeps its type + strong // the value is unchanged +}; + +// Runs op on a fresh value make() with the 0th, 1st, 2nd, ... allocation made +// by the library failing in turn, until op completes. Every failed attempt +// must have thrown std::bad_alloc and left the value valid, with all parent +// pointers intact (copying a value asserts the invariant), and for the strong +// guarantee exactly as it was. +template +void check_exception_guarantee(guarantee g, + const std::function& make, + const std::function& op) +{ + fail_at_alloc_call = -1; + + bool completed = false; + std::size_t failures = 0; + for (std::size_t n = 0; !completed; ++n) + { + CAPTURE(n) + REQUIRE(n < 1000); + + BasicJsonType j = make(); + const std::string before = j.dump(); + const auto type_before = j.type(); + + alloc_call_count = 0; + fail_at_alloc_call = static_cast(n); + try + { + op(j); + completed = true; + } + catch (const std::bad_alloc&) + { + ++failures; + } + fail_at_alloc_call = -1; + + if (!completed) + { + CHECK((j.type() == type_before)); + CHECK(children_name_their_parent(j)); + const BasicJsonType copy(j); // NOLINT(performance-unnecessary-copy-initialization): the copy is the point, it allocates + CHECK(copy.dump() == j.dump()); + if (g == guarantee::strong) + { + CHECK(j.dump() == before); + } + } + } + + // the operation allocates, so at least one attempt must have failed + CHECK(failures > 0); +} + +template +void check_insert_exception_guarantees() +{ + using json_t = BasicJsonType; + using make_t = std::function; + + // strings of this length do not fit into the small string buffer, so + // copying them allocates + const json_t val = std::string(64, 'x'); + const json_t source = json_t::array({std::string(64, 'p'), std::string(64, 'q'), std::string(64, 'r')}); + + for (const bool spare_capacity : + { + false, true + }) + { + CAPTURE(spare_capacity) + + // [ "aaa...", "bbb...", "ccc..." ], with a spare slot if requested + const make_t make = [spare_capacity]() + { + json_t j = json_t::array({std::string(64, 'a'), std::string(64, 'b'), std::string(64, 'c')}); + if (spare_capacity) + { + j.push_back(std::string(64, 'd')); + j.erase(3); + } + return json_t(std::move(j)); + }; + + for (const bool at_end : + { + false, true + }) + { + CAPTURE(at_end) + + SECTION("insert(pos, const basic_json&)") + { + check_exception_guarantee(guarantee::strong, make, [&](json_t& j) + { + j.insert(at_end ? j.end() : j.begin() + 1, val); + }); + } + + SECTION("insert(pos, basic_json&&)") + { + check_exception_guarantee(guarantee::strong, make, [&](json_t& j) + { + j.insert(at_end ? j.end() : j.begin() + 1, json_t(val)); + }); + } + + SECTION("insert(pos, size_type, const basic_json&)") + { + check_exception_guarantee(guarantee::basic, make, [&](json_t& j) + { + j.insert(at_end ? j.end() : j.begin() + 1, 3, val); + }); + } + + SECTION("insert(pos, const_iterator, const_iterator)") + { + check_exception_guarantee(guarantee::basic, make, [&](json_t& j) + { + j.insert(at_end ? j.end() : j.begin() + 1, source.begin(), source.end()); + }); + } + + SECTION("insert(pos, initializer_list_t)") + { + check_exception_guarantee(guarantee::basic, make, [&](json_t& j) + { + j.insert(at_end ? j.end() : j.begin() + 1, {val, val}); + }); + } + } + } + + SECTION("insert(const_iterator, const_iterator) for objects") + { + const json_t members = {{"m", std::string(64, 'm')}, {"n", std::string(64, 'n')}, {"o", std::string(64, 'o')}}; + + SECTION("no member exists yet") + { + check_exception_guarantee(guarantee::basic, []() + { + return json_t({{"a", 1}}); + }, [&](json_t& j) + { + j.insert(members.begin(), members.end()); + }); + } + + SECTION("some members exist already") + { + check_exception_guarantee(guarantee::basic, []() + { + return json_t({{"a", 1}, {"n", std::string(64, 'x')}}); + }, [&](json_t& j) + { + j.insert(members.begin(), members.end()); + }); + } + } +} +} // namespace + +TEST_CASE("insert() provides its documented exception guarantees (#5986)") +{ + SECTION("std::map-backed object_t") + { + check_insert_exception_guarantees>(); + } + + SECTION("ordered_map-backed object_t") + { + check_insert_exception_guarantees>(); + } +} +#endif From bfe69c73c23b70b40ab03c2f764e31302578ba8a Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Mon, 2 Nov 2026 17:48:43 +0100 Subject: [PATCH 27/54] json_view: get() of a boolean throws for the exact number types, as basic_json does (#5992) * json_view: get() of a boolean throws for number_integer_t, number_unsigned_t and number_float_t like basic_json (#5988) basic_json::get() of a boolean throws type_error.302 for exactly number_integer_t, number_unsigned_t and number_float_t (dedicated from_json() overloads) and gives 0/1 for every other arithmetic type. The view returned 0/1 for all of them, contradicting its documentation. The view now throws the same exception (same message, with path/bytes decoration) for these three types. basic_json is unchanged; its behavior is documented and pinned by a test. Signed-off-by: Niels Lohmann * tests: skip throwing json_view bool get() combinations without exceptions (#5988) check_boolean_get_parity() called get() for number_integer_t, number_unsigned_t and number_float_t in its JSON_NOEXCEPTION branch, but a boolean throws type_error.302 for exactly these types, so JSON_THROW aborted test-json_view_cpp11/cpp17 in ci_test_noexceptions. Skip these combinations there; the throwing checks stay in the exceptions-enabled build. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01N5NasAgbDGG28MqXjQvLqp Signed-off-by: Claude * Expect the byte range of the root in the json_view boolean get() test With JSON_DIAGNOSTIC_POSITIONS, the type_error.302 message starts with "(bytes 0-4)"; build the expected text with json_view_test::message(), as the other tests of the root value do. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01N5NasAgbDGG28MqXjQvLqp Signed-off-by: Claude --------- Signed-off-by: Niels Lohmann Signed-off-by: Claude Co-authored-by: Claude --- docs/mkdocs/docs/api/basic_json/get.md | 10 +++ docs/mkdocs/docs/api/basic_json_view/get.md | 3 +- include/nlohmann/detail/view/value.hpp | 31 +++++++- single_include/nlohmann/json_view.hpp | 31 +++++++- tests/src/unit-conversions1.cpp | 38 ++++++++++ tests/src/unit-json_view.cpp | 79 +++++++++++++++++++++ 6 files changed, 185 insertions(+), 7 deletions(-) diff --git a/docs/mkdocs/docs/api/basic_json/get.md b/docs/mkdocs/docs/api/basic_json/get.md index 016595d5f..06e34218a 100644 --- a/docs/mkdocs/docs/api/basic_json/get.md +++ b/docs/mkdocs/docs/api/basic_json/get.md @@ -120,6 +120,16 @@ overload (3). See [Number conversion](../../features/types/number_handling.md#number-conversion) for more information. +!!! note "Booleans as numbers" + + A JSON boolean converts to `#!cpp 0` or `#!cpp 1` for every arithmetic type **except** [`number_integer_t`](number_integer_t.md), + [`number_unsigned_t`](number_unsigned_t.md) and [`number_float_t`](number_float_t.md) (by default `#!cpp std::int64_t`, + `#!cpp std::uint64_t`, and `#!cpp double`), for which `#!cpp get()` throws + [`type_error.302`](../../home/exceptions.md#jsonexceptiontype_error302) ("type must be number, but is boolean"). So + `#!cpp json(true).get()` is `#!cpp 1`, while `#!cpp json(true).get()` and + `#!cpp json(true).get()` throw. This is the behavior of all 3.x versions; + [`basic_json_view::get()`](../basic_json_view/get.md) behaves the same. + !!! note "`std::optional` conversions" Prior to version 3.13.0, `#!cpp get>()` (and other conversions to `std::optional`) failed to diff --git a/docs/mkdocs/docs/api/basic_json_view/get.md b/docs/mkdocs/docs/api/basic_json_view/get.md index 7ee7952bf..1739d372b 100644 --- a/docs/mkdocs/docs/api/basic_json_view/get.md +++ b/docs/mkdocs/docs/api/basic_json_view/get.md @@ -11,7 +11,8 @@ For the types below, the conversion works directly on the flat index — no `Bas - `#!cpp bool` - arithmetic types other than `#!cpp bool` (for a boolean, `#!cpp 0`/`#!cpp 1`, exactly as for - [`BasicJsonType::get()`](../basic_json/get.md)) + [`BasicJsonType::get()`](../basic_json/get.md#notes), except for `number_integer_t`, `number_unsigned_t` and + `number_float_t`, which throw `type_error.302` for a boolean, as `BasicJsonType::get()` does) - `#!cpp std::nullptr_t` - `#!cpp std::basic_string` (including `string_t`) — a copy of the string - [`string_view_t`](index.md#member-types) — **no copy**: the returned view points into the document's diff --git a/include/nlohmann/detail/view/value.hpp b/include/nlohmann/detail/view/value.hpp index ffdcc3605..81b175e2b 100644 --- a/include/nlohmann/detail/view/value.hpp +++ b/include/nlohmann/detail/view/value.hpp @@ -33,12 +33,37 @@ namespace view template struct value_tag {}; +/// whether basic_json reads T with a dedicated from_json() overload that does +/// not accept a boolean: exactly number_integer_t, number_unsigned_t and +/// number_float_t (the other arithmetic types convert a boolean to 0 or 1) +template +struct rejects_boolean : std::integral_constant < bool, + std::is_same::value + || std::is_same::value + || std::is_same::value > +{}; + +/// a boolean as T: 0 or 1 +template +NLOHMANN_VIEW_ALWAYS_INLINE T boolean_as(const document_data& /*d*/, const node& n, std::false_type /*rejects*/) +{ + return static_cast((n.flags & node_flags::is_true) != 0); +} + +/// a boolean as number_integer_t, number_unsigned_t or number_float_t: type_error.302, as in basic_json +template +[[noreturn]] NLOHMANN_VIEW_ALWAYS_INLINE T boolean_as(const document_data& d, const node& n, std::true_type /*rejects*/) +{ + throw_type_error(error_site(&d, &n), exception_id::type_mismatch, "type must be number, but is ", value_type_name(static_cast(n.kind))); +} + /*! @brief the number or boolean of a node converted to an arithmetic type As basic_json's get() for arithmetic types: integers and floats are -converted with static_cast, booleans give 0 or 1, and other types throw -type_error.302. +converted with static_cast, booleans give 0 or 1 (but throw type_error.302 for +number_integer_t, number_unsigned_t and number_float_t, as basic_json does), +and other types throw type_error.302. */ template NLOHMANN_VIEW_ALWAYS_INLINE T arithmetic_value(const document_data& d, const node& n) @@ -52,7 +77,7 @@ NLOHMANN_VIEW_ALWAYS_INLINE T arithmetic_value(const document_data& d, const nod case value_t::number_float: return static_cast(float_value(d, n)); case value_t::boolean: - return static_cast((n.flags & node_flags::is_true) != 0); + return boolean_as(d, n, rejects_boolean {}); case value_t::null: case value_t::object: case value_t::array: diff --git a/single_include/nlohmann/json_view.hpp b/single_include/nlohmann/json_view.hpp index 6bb6ec771..4a7b949a8 100644 --- a/single_include/nlohmann/json_view.hpp +++ b/single_include/nlohmann/json_view.hpp @@ -10072,12 +10072,37 @@ namespace view template struct value_tag {}; +/// whether basic_json reads T with a dedicated from_json() overload that does +/// not accept a boolean: exactly number_integer_t, number_unsigned_t and +/// number_float_t (the other arithmetic types convert a boolean to 0 or 1) +template +struct rejects_boolean : std::integral_constant < bool, + std::is_same::value + || std::is_same::value + || std::is_same::value > +{}; + +/// a boolean as T: 0 or 1 +template +NLOHMANN_VIEW_ALWAYS_INLINE T boolean_as(const document_data& /*d*/, const node& n, std::false_type /*rejects*/) +{ + return static_cast((n.flags & node_flags::is_true) != 0); +} + +/// a boolean as number_integer_t, number_unsigned_t or number_float_t: type_error.302, as in basic_json +template +[[noreturn]] NLOHMANN_VIEW_ALWAYS_INLINE T boolean_as(const document_data& d, const node& n, std::true_type /*rejects*/) +{ + throw_type_error(error_site(&d, &n), exception_id::type_mismatch, "type must be number, but is ", value_type_name(static_cast(n.kind))); +} + /*! @brief the number or boolean of a node converted to an arithmetic type As basic_json's get() for arithmetic types: integers and floats are -converted with static_cast, booleans give 0 or 1, and other types throw -type_error.302. +converted with static_cast, booleans give 0 or 1 (but throw type_error.302 for +number_integer_t, number_unsigned_t and number_float_t, as basic_json does), +and other types throw type_error.302. */ template NLOHMANN_VIEW_ALWAYS_INLINE T arithmetic_value(const document_data& d, const node& n) @@ -10091,7 +10116,7 @@ NLOHMANN_VIEW_ALWAYS_INLINE T arithmetic_value(const document_data& d, const nod case value_t::number_float: return static_cast(float_value(d, n)); case value_t::boolean: - return static_cast((n.flags & node_flags::is_true) != 0); + return boolean_as(d, n, rejects_boolean {}); case value_t::null: case value_t::object: case value_t::array: diff --git a/tests/src/unit-conversions1.cpp b/tests/src/unit-conversions1.cpp index 3d39e10c9..888979440 100644 --- a/tests/src/unit-conversions1.cpp +++ b/tests/src/unit-conversions1.cpp @@ -1260,6 +1260,44 @@ TEST_CASE("value conversion") #undef JSON_HAS_CPP_17 #endif +TEST_CASE("get() of a boolean (#5988)") +{ + // a boolean converts to 0/1 for every arithmetic type except number_integer_t, number_unsigned_t and + // number_float_t, whose dedicated from_json() overloads throw + const json t = true; + const json f = false; + + SECTION("0 and 1") + { + // long and unsigned long are left out: they are std::int64_t and std::uint64_t on some platforms + CHECK(t.get() == true); + CHECK(f.get() == false); + CHECK(t.get() == 1); + CHECK(t.get() == 1); + CHECK(t.get() == 1); + CHECK(t.get() == 1); // NOLINT(google-runtime-int) + CHECK(t.get() == 1); // NOLINT(google-runtime-int) + CHECK(t.get() == 1); + CHECK(t.get() == 1); + CHECK(t.get() == 1.0F); + CHECK(t.get() == 1.0L); + CHECK(f.get() == 0); + CHECK(f.get() == 0.0F); + } + +#if !defined(JSON_NOEXCEPTION) + SECTION("number_integer_t, number_unsigned_t and number_float_t throw") + { + CHECK_THROWS_WITH_AS(t.get(), "[json.exception.type_error.302] type must be number, but is boolean", json::type_error&); + CHECK_THROWS_WITH_AS(t.get(), "[json.exception.type_error.302] type must be number, but is boolean", json::type_error&); + CHECK_THROWS_WITH_AS(t.get(), "[json.exception.type_error.302] type must be number, but is boolean", json::type_error&); + CHECK_THROWS_WITH_AS(f.get(), "[json.exception.type_error.302] type must be number, but is boolean", json::type_error&); + CHECK_THROWS_WITH_AS(f.get(), "[json.exception.type_error.302] type must be number, but is boolean", json::type_error&); + CHECK_THROWS_WITH_AS(f.get(), "[json.exception.type_error.302] type must be number, but is boolean", json::type_error&); + } +#endif +} + #ifdef JSON_HAS_CPP_14 #undef JSON_HAS_CPP_14 #endif diff --git a/tests/src/unit-json_view.cpp b/tests/src/unit-json_view.cpp index c823ef9e2..8dc60e912 100644 --- a/tests/src/unit-json_view.cpp +++ b/tests/src/unit-json_view.cpp @@ -1551,6 +1551,85 @@ TEST_CASE("json_view values") } } +namespace +{ +// get() of a boolean: the same value or the same exception in basic_json and in the view (#5988) +template +void check_boolean_get_parity() +{ + for (const bool b : + { + true, false + }) + { + const std::string text = b ? "true" : "false"; + const json j = json::parse(text); + const json_document d = json_document::parse(text); + CAPTURE(text) +#if !defined(JSON_NOEXCEPTION) + T dom_value {}; + T view_value{}; + const std::string dom_error = json_view_test::exception_of_call([&] { dom_value = j.get(); }); + const std::string view_error = json_view_test::exception_of_call([&] { view_value = d.root().template get(); }); + CHECK(dom_error == view_error); + if (dom_error.empty()) + { + CHECK(dom_value == view_value); + CHECK(view_value == static_cast(b)); + } +#else + // without exceptions, the combinations that throw (number_integer_t, number_unsigned_t, number_float_t) + // cannot be run: JSON_THROW aborts + const bool throws = std::is_same::value + || std::is_same::value + || std::is_same::value; + if (!throws) + { + CHECK(j.get() == d.root().template get()); + } +#endif + } +} +} // namespace + +TEST_CASE("json_view get() of a boolean (#5988)") +{ + check_boolean_get_parity(); + check_boolean_get_parity(); + check_boolean_get_parity(); + check_boolean_get_parity(); + check_boolean_get_parity(); // NOLINT(google-runtime-int) + check_boolean_get_parity(); // NOLINT(google-runtime-int) + check_boolean_get_parity(); + check_boolean_get_parity(); + check_boolean_get_parity(); // NOLINT(google-runtime-int) + check_boolean_get_parity(); // NOLINT(google-runtime-int) + check_boolean_get_parity(); // NOLINT(google-runtime-int) + check_boolean_get_parity(); // NOLINT(google-runtime-int) + check_boolean_get_parity(); + check_boolean_get_parity(); + check_boolean_get_parity(); + check_boolean_get_parity(); + check_boolean_get_parity(); + +#if !defined(JSON_NOEXCEPTION) + SECTION("the exceptions of the exact number types") + { + const json_document d = json_document::parse("true"); + CHECK_THROWS_WITH_AS(d.root().get(), json_view_test::message("type_error.302", "", 0, 4, "type must be number, but is boolean").c_str(), json::type_error&); + CHECK_THROWS_WITH_AS(d.root().get(), json_view_test::message("type_error.302", "", 0, 4, "type must be number, but is boolean").c_str(), json::type_error&); + CHECK_THROWS_WITH_AS(d.root().get(), json_view_test::message("type_error.302", "", 0, 4, "type must be number, but is boolean").c_str(), json::type_error&); + CHECK(d.root().get() == 1); + // get_to() and value() go through get() + json::number_float_t x = 0; + CHECK_THROWS_AS(d.root().get_to(x), json::type_error&); + const json_document o = json_document::parse("{\"b\":true}"); + CHECK_THROWS_AS(static_cast(o.root().value("b", 0.0)), json::type_error&); + CHECK(o.root().value("b", 0) == 1); + } +#endif +} + TEST_CASE("json_view JSON pointers") { SECTION("every value of generated documents") From 732831013699a38d80f3e1933e64e0e14f76528e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Nov 2026 17:48:55 +0100 Subject: [PATCH 28/54] Bump mkdocs-material from 9.7.7 to 9.8.0 in /docs/mkdocs (#6010) Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 9.7.7 to 9.8.0. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/CHANGELOG) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/9.7.7...9.8.0) --- updated-dependencies: - dependency-name: mkdocs-material dependency-version: 9.8.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- docs/mkdocs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/mkdocs/requirements.txt b/docs/mkdocs/requirements.txt index 245ebc387..dc260875f 100644 --- a/docs/mkdocs/requirements.txt +++ b/docs/mkdocs/requirements.txt @@ -2,7 +2,7 @@ wheel==0.48.0 mkdocs==1.6.1 # documentation framework mkdocs-git-revision-date-localized-plugin==1.6.0 # plugin "git-revision-date-localized" -mkdocs-material==9.7.7 # theme for mkdocs +mkdocs-material==9.8.0 # theme for mkdocs mkdocs-material-extensions==1.3.1 # extensions mkdocs-minify-plugin==0.8.0 # plugin "minify" mkdocs-redirects==1.2.3 # plugin "redirects" From ff0bdfdef3ae115c9ee9e7ee053ad17119e1aced Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Mon, 2 Nov 2026 17:50:40 +0100 Subject: [PATCH 29/54] Keep the byte range in exceptions about the root value (#5990) * Keep the byte range in exceptions about the root value (#5987) With JSON_DIAGNOSTICS and JSON_DIAGNOSTIC_POSITIONS both set, exception::diagnostics() and the json_view diagnostics() returned early for a value without a path (the root), dropping "(bytes a-b)". Build the "(path) " prefix only when there are tokens and always append the byte range. Signed-off-by: Niels Lohmann * Guard the null value of a value-initialized iterator; drop the unneeded found flag With the root's byte range, diagnostics() reaches get_byte_positions() also with JSON_DIAGNOSTICS when there is no value, as for a value-initialized iterator (invalid_iterator.212). get_byte_positions() now returns no range for a null value, the same change as #6003, so either can be merged first. In the view, a value that value_path() does not find leaves no tokens, so the found flag added nothing to the tokens.empty() test. Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann --- .../api/macros/json_diagnostic_positions.md | 3 ++ include/nlohmann/detail/exceptions.hpp | 19 ++++++------ include/nlohmann/detail/view/errors.hpp | 25 ++++++++-------- single_include/nlohmann/json.hpp | 19 ++++++------ single_include/nlohmann/json_view.hpp | 25 ++++++++-------- tests/src/unit-diagnostic-positions.cpp | 22 ++++++++++++++ tests/src/unit-json_view_positions.cpp | 29 +++++++++++++++---- 7 files changed, 92 insertions(+), 50 deletions(-) diff --git a/docs/mkdocs/docs/api/macros/json_diagnostic_positions.md b/docs/mkdocs/docs/api/macros/json_diagnostic_positions.md index 8a772791a..4dac9f9ff 100644 --- a/docs/mkdocs/docs/api/macros/json_diagnostic_positions.md +++ b/docs/mkdocs/docs/api/macros/json_diagnostic_positions.md @@ -35,6 +35,9 @@ Note that enabling this macro increases the size of every JSON value by two `std overhead to parsing, copying JSON value objects, and the generation of error messages for exceptions. It also causes these values to be reported in those error messages. +If [`JSON_DIAGNOSTICS`](json_diagnostics.md) is also set, the message contains the path and the byte range, such as +`(/1) (bytes 4-8)`. An exception about the root value has no path and only the range, such as `(bytes 0-4)`. + ## Default definition The default value is `0` (position diagnostics are switched off). diff --git a/include/nlohmann/detail/exceptions.hpp b/include/nlohmann/detail/exceptions.hpp index 098d70495..fc9359882 100644 --- a/include/nlohmann/detail/exceptions.hpp +++ b/include/nlohmann/detail/exceptions.hpp @@ -198,18 +198,19 @@ class exception : public std::exception // all other parent types (primitives) contribute no token } - if (tokens.empty()) + // the root value has no path, but still has a byte range + std::string prefix; + if (!tokens.empty()) { - return ""; + const auto str = std::accumulate(tokens.rbegin(), tokens.rend(), std::string{}, + [](const std::string & a, const std::string & b) + { + return concat(a, '/', detail::escape(b)); + }); + prefix = concat('(', str, ") "); } - auto str = std::accumulate(tokens.rbegin(), tokens.rend(), std::string{}, - [](const std::string & a, const std::string & b) - { - return concat(a, '/', detail::escape(b)); - }); - - return concat('(', str, ") ", get_byte_positions(leaf_element)); + return concat(prefix, get_byte_positions(leaf_element)); #else return get_byte_positions(leaf_element); #endif diff --git a/include/nlohmann/detail/view/errors.hpp b/include/nlohmann/detail/view/errors.hpp index eeab3bbd5..64c514738 100644 --- a/include/nlohmann/detail/view/errors.hpp +++ b/include/nlohmann/detail/view/errors.hpp @@ -234,8 +234,7 @@ void value_range(const document_data& d, const node* n, std::size_t& start, std: /// the context basic_json puts in front of the message of an exception about /// a value: "(/path) " with JSON_DIAGNOSTICS (none for the root), then -/// "(bytes a-b) " with JSON_DIAGNOSTIC_POSITIONS (with JSON_DIAGNOSTICS, only -/// if there is a path) +/// "(bytes a-b) " with JSON_DIAGNOSTIC_POSITIONS (also for the root) /// /// This and the throw helpers that take an error_site are templates, so that /// the path and range code is instantiated only where an exception can be @@ -256,19 +255,19 @@ std::string diagnostics(const error_site site) std::string context; #if JSON_DIAGNOSTICS std::vector tokens; - const bool found = edited ? value_path(d, site.value(), tokens) : value_path(d, site.value(), tokens); - if (!found || tokens.empty()) + // a value that is not found leaves no tokens, like the root + static_cast(edited ? value_path(d, site.value(), tokens) : value_path(d, site.value(), tokens)); + if (!tokens.empty()) { - // context is still empty; returning it keeps the copy elided (-Wnrvo) - return context; + // the root value has no path, but still has a byte range + std::string path; + for (const std::string& token : tokens) + { + path.push_back('/'); + path += detail::escape(token); + } + context = concat('(', path, ") "); } - std::string path; - for (const std::string& token : tokens) - { - path.push_back('/'); - path += detail::escape(token); - } - context = concat('(', path, ") "); #endif #if JSON_DIAGNOSTIC_POSITIONS std::size_t start = 0; diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 80f938972..7029ed34e 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -5695,18 +5695,19 @@ class exception : public std::exception // all other parent types (primitives) contribute no token } - if (tokens.empty()) + // the root value has no path, but still has a byte range + std::string prefix; + if (!tokens.empty()) { - return ""; + const auto str = std::accumulate(tokens.rbegin(), tokens.rend(), std::string{}, + [](const std::string & a, const std::string & b) + { + return concat(a, '/', detail::escape(b)); + }); + prefix = concat('(', str, ") "); } - auto str = std::accumulate(tokens.rbegin(), tokens.rend(), std::string{}, - [](const std::string & a, const std::string & b) - { - return concat(a, '/', detail::escape(b)); - }); - - return concat('(', str, ") ", get_byte_positions(leaf_element)); + return concat(prefix, get_byte_positions(leaf_element)); #else return get_byte_positions(leaf_element); #endif diff --git a/single_include/nlohmann/json_view.hpp b/single_include/nlohmann/json_view.hpp index 4a7b949a8..0861bd95e 100644 --- a/single_include/nlohmann/json_view.hpp +++ b/single_include/nlohmann/json_view.hpp @@ -5076,8 +5076,7 @@ void value_range(const document_data& d, const node* n, std::size_t& start, std: /// the context basic_json puts in front of the message of an exception about /// a value: "(/path) " with JSON_DIAGNOSTICS (none for the root), then -/// "(bytes a-b) " with JSON_DIAGNOSTIC_POSITIONS (with JSON_DIAGNOSTICS, only -/// if there is a path) +/// "(bytes a-b) " with JSON_DIAGNOSTIC_POSITIONS (also for the root) /// /// This and the throw helpers that take an error_site are templates, so that /// the path and range code is instantiated only where an exception can be @@ -5098,19 +5097,19 @@ std::string diagnostics(const error_site site) std::string context; #if JSON_DIAGNOSTICS std::vector tokens; - const bool found = edited ? value_path(d, site.value(), tokens) : value_path(d, site.value(), tokens); - if (!found || tokens.empty()) + // a value that is not found leaves no tokens, like the root + static_cast(edited ? value_path(d, site.value(), tokens) : value_path(d, site.value(), tokens)); + if (!tokens.empty()) { - // context is still empty; returning it keeps the copy elided (-Wnrvo) - return context; + // the root value has no path, but still has a byte range + std::string path; + for (const std::string& token : tokens) + { + path.push_back('/'); + path += detail::escape(token); + } + context = concat('(', path, ") "); } - std::string path; - for (const std::string& token : tokens) - { - path.push_back('/'); - path += detail::escape(token); - } - context = concat('(', path, ") "); #endif #if JSON_DIAGNOSTIC_POSITIONS std::size_t start = 0; diff --git a/tests/src/unit-diagnostic-positions.cpp b/tests/src/unit-diagnostic-positions.cpp index 11f63fb59..85284d9d4 100644 --- a/tests/src/unit-diagnostic-positions.cpp +++ b/tests/src/unit-diagnostic-positions.cpp @@ -193,6 +193,28 @@ TEST_CASE("Better diagnostics with positions") check_conversion(300); } + SECTION("exception about the root value (#5987)") + { + // the root has no path, but still has a byte range + const json root = json::parse("true"); + const json nested = json::parse("[1, true]"); + const json constructed = true; +#if JSON_DIAGNOSTICS + CHECK_THROWS_WITH_AS(root.get(), + "[json.exception.type_error.302] (bytes 0-4) type must be string, but is boolean", json::type_error); + CHECK_THROWS_WITH_AS(nested[1].get(), + "[json.exception.type_error.302] (/1) (bytes 4-8) type must be string, but is boolean", json::type_error); +#else + CHECK_THROWS_WITH_AS(root.get(), + "[json.exception.type_error.302] (bytes 0-4) type must be string, but is boolean", json::type_error); + CHECK_THROWS_WITH_AS(nested[1].get(), + "[json.exception.type_error.302] (bytes 4-8) type must be string, but is boolean", json::type_error); +#endif + // a value without positions gets no range and no stray whitespace + CHECK_THROWS_WITH_AS(constructed.get(), + "[json.exception.type_error.302] type must be string, but is boolean", json::type_error); + } + SECTION("JSON patch add to primitive parent (#4292)") { // the JSON Patch "add" target /foo/bar/baz has a string parent diff --git a/tests/src/unit-json_view_positions.cpp b/tests/src/unit-json_view_positions.cpp index 966c4b46a..d2e951cef 100644 --- a/tests/src/unit-json_view_positions.cpp +++ b/tests/src/unit-json_view_positions.cpp @@ -300,6 +300,27 @@ TEST_CASE("json_view materialize() source positions") static_cast(j.at("list").at(1).get()); })); } + + SECTION("exception about the root value (#5987)") + { + const json_document root = json_document::parse("true"); + const json_document nested = json_document::parse("[1, true]"); + CHECK(json_view_test::exception_of_call([&] + { + static_cast(root.root().get()); + }) == "[json.exception.type_error.302] (bytes 0-4) type must be string, but is boolean"); +#if JSON_DIAGNOSTICS + CHECK(json_view_test::exception_of_call([&] + { + static_cast(nested.root()[1].get()); + }) == "[json.exception.type_error.302] (/1) (bytes 4-8) type must be string, but is boolean"); +#else + CHECK(json_view_test::exception_of_call([&] + { + static_cast(nested.root()[1].get()); + }) == "[json.exception.type_error.302] (bytes 4-8) type must be string, but is boolean"); +#endif + } #endif SECTION("edited documents") @@ -551,12 +572,8 @@ TEST_CASE("json_view exceptions name the path and the bytes of the value") expected_zz += "(bytes 12-35) key 'zz' not found"; CHECK(exception_of_call([&] { static_cast(d.root().at(json::json_pointer("/people/0/zz"))); }) == expected_zz); - // the root has no path, and with JSON_DIAGNOSTICS then no bytes either (as basic_json) - std::string expected_missing = "[json.exception.out_of_range.403] "; -#if !JSON_DIAGNOSTICS - expected_missing += "(bytes 0-79) "; -#endif - expected_missing += "key 'missing' not found"; + // the root has no path, but still its bytes (#5987) + const std::string expected_missing = "[json.exception.out_of_range.403] (bytes 0-79) key 'missing' not found"; CHECK(exception_of_call([&] { static_cast(d.root().at("missing")); }) == expected_missing); } From 1d69ef9ab41a8e30e8ba0cdb6e253260ffc3f319 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Mon, 2 Nov 2026 17:54:38 +0100 Subject: [PATCH 30/54] Raise line coverage to 99.9 %: fix the measurement, remove unreachable code, add tests (#5997) * CI: measure line coverage without lcov's unexecuted-block adjustment Drop geninfo_unexecuted_blocks=1 from the lcov capture of ci_test_coverage: it reset every line of an inlined or templated function to 0 as soon as one copy had an unexecuted block, which understated the line coverage by about three points. Add -fvar-tracking-assignments to both coverage builds. At -O0, GCC inlines always_inline functions before it places the coverage counters, so the call sites of such helpers (put_literal() and others) got no line record and showed as 0 in an -O3 translation unit that never reaches them. With the flag, the call sites carry counts in every translation unit. The generated code is identical. Signed-off-by: Niels Lohmann * Remove unreachable code found by the coverage analysis Remove code that no input can reach, and mark the lines that cannot be covered with LCOV_EXCL_LINE: - BJData writer: the dry_run parameter of the ND-array element writers was always true; replace them by range checks. - BON8 writer: call throw_integer_out_of_range() directly instead of write_bon8_value() for values that always throw. - binary_writer: integer_bits() is only called for integers of up to 64 bits. - serializer, view serializer, lexer: replace guards that the callers make unreachable by assertions or by the simpler code (surrogate pairs under ensure_ascii, next_unget, '-' at the end of the buffer). - json_view: drop unreachable branches in scalar(), set(), check_utf8_scan(), append_input_impl() and unescape(). - Mark exception-cleanup braces, 2 GiB limits, big-endian and 32-bit code, and fall-through case labels with LCOV_EXCL_LINE. No behavior changes for any input that reaches these functions. Signed-off-by: Niels Lohmann * Tests: cover the remaining lines of the library headers Add tests for the lines of include/nlohmann that were not covered: the binary readers and writers, the lexer and locale paths, the json_view modules, the custom number, key and array types, json_pointer, json_patch, diff and the iterators. Signed-off-by: Niels Lohmann * Fix test failures in the coverage tests - unit-custom-object-key-type: remove the unused make_strict_key(strict_key) overload, the unused range constructor and default constructor, and guard data() with JSON_DIAGNOSTICS (unused-function/-member-function errors with GCC and clang -Weverything). - unit-json_view_compare: move the two new sections to the new unit-json_view_compare_values.cpp, as the object file got too large for the MinGW assembler again, and use a std::map with std::greater instead of std::unordered_map, which does not accept the incomplete basic_json (clang 3.9, icpc). - unit-json_view_dump: rename the variables that hid outer ones (MSVC C4456). - unit-stream-bulk-scan: restore the real size of the oversized get area before the adapter is destroyed (JSON_PRECISE_STREAM_POSITION reads the character at the read position), and skip the test where the streambuf cannot hold a get area larger than an int (MSVC). - unit-regression1: with JSON_NO_THREAD_LOCAL, copies are built member by member and keep correct timestamps, so check the corresponding outcome. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01N5NasAgbDGG28MqXjQvLqp Signed-off-by: Claude * Return std::size_t from the hash helper in the ordered_json test MSVC (Win32) rejected the implicit conversion of the unsigned long long product to the 32-bit std::size_t (C4244 as an error). Also note when JSON_NO_THREAD_LOCAL is set in the regression test. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01N5NasAgbDGG28MqXjQvLqp Signed-off-by: Claude --------- Signed-off-by: Niels Lohmann Signed-off-by: Claude Co-authored-by: Claude --- cmake/ci.cmake | 6 +- .../nlohmann/detail/conversions/from_json.hpp | 2 +- .../nlohmann/detail/input/input_adapters.hpp | 2 +- include/nlohmann/detail/input/json_sax.hpp | 2 +- include/nlohmann/detail/input/lexer.hpp | 23 +- include/nlohmann/detail/input/parser.hpp | 2 +- include/nlohmann/detail/json_pointer.hpp | 8 +- .../nlohmann/detail/output/binary_writer.hpp | 152 ++++------- include/nlohmann/detail/output/serializer.hpp | 14 +- include/nlohmann/detail/string_concat.hpp | 2 +- include/nlohmann/detail/string_escape.hpp | 4 - include/nlohmann/detail/string_utils.hpp | 6 +- include/nlohmann/detail/view/compare.hpp | 27 +- include/nlohmann/detail/view/edit.hpp | 4 +- include/nlohmann/detail/view/edit_storage.hpp | 2 +- include/nlohmann/detail/view/errors.hpp | 2 + include/nlohmann/detail/view/materialize.hpp | 11 +- include/nlohmann/detail/view/serializer.hpp | 24 +- include/nlohmann/json.hpp | 26 +- include/nlohmann/json_view.hpp | 6 +- single_include/nlohmann/json.hpp | 243 +++++++----------- single_include/nlohmann/json_view.hpp | 76 ++---- tests/src/unit-binary_writer_sinks.cpp | 68 +++++ tests/src/unit-bjdata.cpp | 26 ++ tests/src/unit-bon8.cpp | 3 + tests/src/unit-bson.cpp | 75 ++++++ tests/src/unit-cbor.cpp | 24 ++ tests/src/unit-class_lexer.cpp | 75 ++++++ tests/src/unit-custom-array-type.cpp | 20 ++ tests/src/unit-custom-key-type-full.cpp | 21 ++ tests/src/unit-custom-number-types.cpp | 66 +++++ tests/src/unit-custom-object-key-type.cpp | 163 ++++++++++++ tests/src/unit-deserialization.cpp | 22 ++ tests/src/unit-diagnostics.cpp | 20 ++ tests/src/unit-iterators2.cpp | 19 ++ tests/src/unit-json_patch.cpp | 98 +++++++ tests/src/unit-json_pointer.cpp | 16 ++ tests/src/unit-json_view.cpp | 52 +++- tests/src/unit-json_view_compare_values.cpp | 83 ++++++ tests/src/unit-json_view_dump.cpp | 27 ++ tests/src/unit-json_view_dump_escape.cpp | 40 +++ tests/src/unit-json_view_edit.cpp | 17 ++ tests/src/unit-json_view_from_json.cpp | 11 + tests/src/unit-json_view_image.cpp | 7 + tests/src/unit-json_view_large.cpp | 56 ++++ tests/src/unit-json_view_positions.cpp | 70 +++++ tests/src/unit-large_json.cpp | 53 ++++ tests/src/unit-locale-cpp.cpp | 203 +++++++++++++++ tests/src/unit-msgpack-cpp17.cpp | 30 +++ tests/src/unit-ordered_json.cpp | 86 +++++++ tests/src/unit-regression1.cpp | 72 ++++++ tests/src/unit-regression2.cpp | 4 + tests/src/unit-stream-bulk-scan.cpp | 58 +++++ tests/src/unit-to_chars.cpp | 46 ++++ tests/src/unit-ubjson.cpp | 25 ++ 55 files changed, 1892 insertions(+), 408 deletions(-) create mode 100644 tests/src/unit-json_view_compare_values.cpp diff --git a/cmake/ci.cmake b/cmake/ci.cmake index 0d4b3db92..b41ab8b62 100644 --- a/cmake/ci.cmake +++ b/cmake/ci.cmake @@ -348,20 +348,20 @@ add_custom_target(ci_test_no_thread_local add_custom_target(ci_test_coverage COMMAND CXX=g++ ${CMAKE_COMMAND} - -DCMAKE_BUILD_TYPE=Debug -GNinja -DCMAKE_CXX_FLAGS="--coverage;-fprofile-arcs;-ftest-coverage" + -DCMAKE_BUILD_TYPE=Debug -GNinja -DCMAKE_CXX_FLAGS="--coverage;-fprofile-arcs;-ftest-coverage;-fvar-tracking-assignments" -DJSON_BuildTests=ON -S${PROJECT_SOURCE_DIR} -B${PROJECT_BINARY_DIR}/build_coverage COMMAND ${CMAKE_COMMAND} --build ${PROJECT_BINARY_DIR}/build_coverage COMMAND cd ${PROJECT_BINARY_DIR}/build_coverage && ${CMAKE_CTEST_COMMAND} --parallel ${N} --output-on-failure COMMAND CXX=g++ ${CMAKE_COMMAND} - -DCMAKE_BUILD_TYPE=Debug -GNinja -DCMAKE_CXX_FLAGS="-m32;--coverage;-fprofile-arcs;-ftest-coverage" + -DCMAKE_BUILD_TYPE=Debug -GNinja -DCMAKE_CXX_FLAGS="-m32;--coverage;-fprofile-arcs;-ftest-coverage;-fvar-tracking-assignments" -DJSON_BuildTests=ON -DJSON_32bitTest=ONLY -S${PROJECT_SOURCE_DIR} -B${PROJECT_BINARY_DIR}/build_coverage32 COMMAND ${CMAKE_COMMAND} --build ${PROJECT_BINARY_DIR}/build_coverage32 COMMAND cd ${PROJECT_BINARY_DIR}/build_coverage32 && ${CMAKE_CTEST_COMMAND} --parallel ${N} --output-on-failure - COMMAND ${LCOV_TOOL} --directory . --capture --output-file json.info --rc branch_coverage=1 --rc geninfo_unexecuted_blocks=1 --ignore-errors mismatch --ignore-errors unused --ignore-errors inconsistent + COMMAND ${LCOV_TOOL} --directory . --capture --output-file json.info --rc branch_coverage=1 --ignore-errors mismatch --ignore-errors unused --ignore-errors inconsistent COMMAND ${LCOV_TOOL} -e json.info ${SRC_FILES} --output-file json.info.filtered --rc branch_coverage=1 --ignore-errors unused --ignore-errors inconsistent COMMAND ${CMAKE_SOURCE_DIR}/tests/thirdparty/imapdl/filterbr.py json.info.filtered > json.info.filtered.noexcept COMMAND ${GENHTML_TOOL} --title "JSON for Modern C++" --legend --demangle-cpp --output-directory html --show-details --branch-coverage --ignore-errors inconsistent json.info.filtered.noexcept diff --git a/include/nlohmann/detail/conversions/from_json.hpp b/include/nlohmann/detail/conversions/from_json.hpp index 8fb9d149b..09f0713ea 100644 --- a/include/nlohmann/detail/conversions/from_json.hpp +++ b/include/nlohmann/detail/conversions/from_json.hpp @@ -91,7 +91,7 @@ const typename BasicJsonType::binary_t& binary_value(const BasicJsonType& j) } template::value, int> = 0> -typename View::basic_json_type::binary_t binary_value(const View& /*unused*/) +typename View::basic_json_type::binary_t binary_value(const View& /*unused*/) // LCOV_EXCL_LINE { return {}; // LCOV_EXCL_LINE } diff --git a/include/nlohmann/detail/input/input_adapters.hpp b/include/nlohmann/detail/input/input_adapters.hpp index af1a6d537..e9cdb7e09 100644 --- a/include/nlohmann/detail/input/input_adapters.hpp +++ b/include/nlohmann/detail/input/input_adapters.hpp @@ -572,7 +572,7 @@ class input_stream_adapter } else { - current_in_get_area = true; + current_in_get_area = true; // LCOV_EXCL_LINE (called only when the get area is exhausted) } #endif return res; diff --git a/include/nlohmann/detail/input/json_sax.hpp b/include/nlohmann/detail/input/json_sax.hpp index 0ceec61b7..7c6a947a7 100644 --- a/include/nlohmann/detail/input/json_sax.hpp +++ b/include/nlohmann/detail/input/json_sax.hpp @@ -999,7 +999,7 @@ class json_sax_dom_parser : public json_sax_dom_error_state ordered_data.members[f.ordered.dest_member].second = std::move(value); } - static void place_member(const frame& /*f*/, BasicJsonType&& /*value*/, std::false_type /*indexed*/) noexcept {} + static void place_member(const frame& /*f*/, BasicJsonType&& /*value*/, std::false_type /*indexed*/) noexcept {} // LCOV_EXCL_LINE /// store the members of a new object directly in it bool start_stored_object(std::size_t len) diff --git a/include/nlohmann/detail/input/lexer.hpp b/include/nlohmann/detail/input/lexer.hpp index 66cb14b4f..7811c9951 100644 --- a/include/nlohmann/detail/input/lexer.hpp +++ b/include/nlohmann/detail/input/lexer.hpp @@ -437,11 +437,10 @@ class lexer : public lexer_base template char_int_type get_after_string_run(std::true_type /*bulk*/, std::integral_constant keep) { - // a pending unget must be consumed through the normal path first - if (next_unget) - { - return get(); - } + // scan_string() is entered from scan_after_whitespace(), whose current + // was read by skip_whitespace() (its first get() clears a pending + // unget), and nothing in scan_string() calls unget() + JSON_ASSERT(!next_unget); const std::size_t remaining = ia.bulk_remaining(); if (remaining == 0) { @@ -1516,11 +1515,9 @@ scan_number_done: */ token_type scan_number_bulk_contiguous() { - // a pending unget offsets the buffer position from current; fall back - if (next_unget) - { - return token_type::uninitialized; - } + // called right after scan_after_whitespace() read current (see + // get_after_string_run()), so no unget is pending + JSON_ASSERT(!next_unget); const std::size_t rem = ia.bulk_remaining(); if (rem == 0) { @@ -1541,11 +1538,7 @@ scan_number_done: if (data[0] == '-') { number_type = token_type::value_integer; - i = 1; - if (i >= avail) - { - return token_type::uninitialized; - } + i = 1; // avail >= 2: rem == 0 returned above } const std::size_t integer_begin = i; if (data[i] == '0') diff --git a/include/nlohmann/detail/input/parser.hpp b/include/nlohmann/detail/input/parser.hpp index 023327166..5e0995b05 100644 --- a/include/nlohmann/detail/input/parser.hpp +++ b/include/nlohmann/detail/input/parser.hpp @@ -598,7 +598,7 @@ class parser } return error_msg; - } + } // LCOV_EXCL_LINE (exception cleanup only) private: /// callback function diff --git a/include/nlohmann/detail/json_pointer.hpp b/include/nlohmann/detail/json_pointer.hpp index 6df49fbc0..ec30e17c5 100644 --- a/include/nlohmann/detail/json_pointer.hpp +++ b/include/nlohmann/detail/json_pointer.hpp @@ -304,7 +304,7 @@ class json_pointer result.reference_tokens.insert(result.reference_tokens.end(), lhs.reference_tokens.begin(), lhs.reference_tokens.end()); result.reference_tokens.insert(result.reference_tokens.end(), rhs.reference_tokens.begin(), rhs.reference_tokens.end()); return result; - } + } // LCOV_EXCL_LINE (exception cleanup only) /// @brief create a new JSON pointer by appending the unescaped token at the end of the JSON pointer /// @sa https://json.nlohmann.me/api/json_pointer/operator_slash/ @@ -315,7 +315,7 @@ class json_pointer result.reference_tokens.insert(result.reference_tokens.end(), lhs.reference_tokens.begin(), lhs.reference_tokens.end()); result.reference_tokens.push_back(std::move(token)); return result; - } + } // LCOV_EXCL_LINE (exception cleanup only) /// @brief create a new JSON pointer by appending the array-index-token at the end of the JSON pointer /// @sa https://json.nlohmann.me/api/json_pointer/operator_slash/ @@ -2180,7 +2180,7 @@ class json_pointer { // create an entry in the array current = ¤t->operator[](array_index(token)); - break; + break; // LCOV_EXCL_LINE (needs an array index with 10 or more digits: more than 1e9 elements) } /* @@ -2217,7 +2217,7 @@ class json_pointer json_pointer result; result.reference_tokens = reference_tokens; return result; - } + } // LCOV_EXCL_LINE (exception cleanup only) json_pointer convert()&& { diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index c9bdc696f..5bbaa42d2 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -833,7 +833,7 @@ class binary_writer while (n != 0); std::reverse(result.begin(), result.end()); return result; - } + } // LCOV_EXCL_LINE (exception cleanup only) /*! @brief the decimal digits of the integer @a n, with a leading '-' if it @@ -860,7 +860,7 @@ class binary_writer } std::reverse(result.begin(), result.end()); return result; - } + } // LCOV_EXCL_LINE (exception cleanup only) /*! @brief throws because the integer @a j is too large for @a format @@ -1792,7 +1792,7 @@ class binary_writer std::memcpy(out, &n, sizeof(NumberType)); if (!is_little_endian) { - std::reverse(out, out + sizeof(NumberType)); + std::reverse(out, out + sizeof(NumberType)); // LCOV_EXCL_LINE (big-endian hosts only) } } @@ -1921,7 +1921,7 @@ class binary_writer const string_t& value = *j.m_data.m_value.string; if (JSON_HEDLEY_UNLIKELY(!value_in_range_of(value.size() + 1ul))) { - return false; + return false; // LCOV_EXCL_LINE (a string of 2 GiB or more) } out[0] = to_char_type(0x02); store_little_endian(p, static_cast(value.size() + 1ul)); @@ -2378,7 +2378,7 @@ class binary_writer const std::size_t size = oa.written() - pos; if (JSON_HEDLEY_UNLIKELY(!value_in_range_of(size))) { - return false; + return false; // LCOV_EXCL_LINE (a document of 2 GiB or more) } const auto le_size = static_cast(size); oa.overwrite(pos, is_little_endian ? le_size : byteswap(le_size)); @@ -2506,7 +2506,7 @@ class binary_writer { if (JSON_HEDLEY_UNLIKELY(!close_bson_container(nested_size_pos))) { - return false; + return false; // LCOV_EXCL_LINE (a document of 2 GiB or more) } continue; } @@ -2521,7 +2521,7 @@ class binary_writer if (JSON_HEDLEY_UNLIKELY(!close_bson_container(current.size_slot))) { - return false; + return false; // LCOV_EXCL_LINE (a document of 2 GiB or more) } if (parents.empty()) { @@ -2778,10 +2778,10 @@ class binary_writer Selected through masks rather than branches, as the type and the sign of the values in an array of mixed signs alternate unpredictably (non-negative - integers are number_unsigned after parsing). Up to 64 bits, the sign is bit - 63 of the result (a narrower integer is sign-extended); computing it with - masks keeps GCC from branching on the type. A wider integer has to be - compared. + integers are number_unsigned after parsing). The sign is bit 63 of the + result (a narrower integer is sign-extended); computing it with masks keeps + GCC from branching on the type. Only called if @ref integers_fit_64_bits(); + a wider number type is written with range checks on the whole value. */ static std::uint64_t integer_bits(const BasicJsonType& j, std::uint64_t& sign_mask) noexcept { @@ -2789,9 +2789,7 @@ class binary_writer const std::uint64_t raw = is_signed ? static_cast(j.m_data.m_value.number_integer) : static_cast(j.m_data.m_value.number_unsigned); - sign_mask = sizeof(typename BasicJsonType::number_integer_t) <= sizeof(std::uint64_t) - ? (std::uint64_t{0} - static_cast(is_signed)) & (std::uint64_t{0} - (raw >> 63u)) - : std::uint64_t{0} - static_cast(is_signed && j.m_data.m_value.number_integer < 0); + sign_mask = (std::uint64_t{0} - static_cast(is_signed)) & (std::uint64_t{0} - (raw >> 63u)); return raw; } @@ -4158,57 +4156,34 @@ class binary_writer } /*! - @brief validate (dry_run) or write one BJData ND-array element of integer dtype @a T - @return whether @a el's value is in range of @a T; always true when @a dry_run is false - */ - template - bool write_bjdata_ndarray_element(const BasicJsonType& el, const bool dry_run) - { - if (dry_run) - { - return bjdata_ndarray_value_in_range(el); - } - using storage_type = typename std::conditional::value, std::uint64_t, std::int64_t>::type; - write_number(static_cast(el.template get()), true); - return true; - } - - /*! - @brief validate (dry_run) or write one BJData ND-array element of dtype 'd' (single precision) + @brief whether the BJData ND-array element @a el can be written with dtype 'd' (single precision) @return whether @a el's value survives narrowing to float and back without any change - (so the ND-array round-trips exactly), or is infinite or NaN; always true when - @a dry_run is false + (so the ND-array round-trips exactly), or is infinite or NaN */ - bool write_bjdata_ndarray_float_element(const BasicJsonType& el, const bool dry_run) + static bool bjdata_ndarray_float_in_range(const BasicJsonType& el) noexcept { const auto dval = static_cast(el.m_data.m_value.number_float); - if (dry_run) - { #ifdef __GNUC__ - JSON_HEDLEY_DIAGNOSTIC_PUSH - JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") + JSON_HEDLEY_DIAGNOSTIC_PUSH + JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") #endif - // a value that would be rounded (rather than exactly represented) by the - // narrowing to float is treated like an out-of-range integer element; this - // is the same criterion write_compact_float() uses for CBOR/MessagePack - const bool in_range = std::isnan(dval) || - (dval >= static_cast(std::numeric_limits::lowest()) && - dval <= static_cast((std::numeric_limits::max)()) && - static_cast(static_cast(dval)) == dval) || - std::isinf(dval); + // a value that would be rounded (rather than exactly represented) by the + // narrowing to float is treated like an out-of-range integer element; this + // is the same criterion write_compact_float() uses for CBOR/MessagePack + return std::isnan(dval) || + (dval >= static_cast(std::numeric_limits::lowest()) && + dval <= static_cast((std::numeric_limits::max)()) && + static_cast(static_cast(dval)) == dval) || + std::isinf(dval); #ifdef __GNUC__ - JSON_HEDLEY_DIAGNOSTIC_POP + JSON_HEDLEY_DIAGNOSTIC_POP #endif - return in_range; - } - write_number(static_cast(dval), true); - return true; } /*! @brief write the elements of an ND-array as the number kind @a dtype names (see @ref write_bjdata_ndarray), after they have been found to be - in range by a dry run of @ref write_bjdata_ndarray_elements + in range by @ref bjdata_ndarray_elements_in_range The elements are encoded in blocks into space reserved from the sink. */ @@ -4290,20 +4265,13 @@ class binary_writer } /*! - @brief validate or write every element of a BJData ND-array's `_ArrayData_` + @brief whether every element of a BJData ND-array's `_ArrayData_` is in range for its dtype @param[in] array_data the `_ArrayData_` array @param[in] dtype the ND-array dtype marker, as returned by bjdata_ndarray_type_marker() - @param[in] dry_run true to only range-check each element, false to write it - @return whether every element is in range for @a dtype (always true when @a dry_run is false) + @return whether every element is in range for @a dtype */ - bool write_bjdata_ndarray_elements(const BasicJsonType& array_data, const CharType dtype, const bool dry_run) + static bool bjdata_ndarray_elements_in_range(const BasicJsonType& array_data, const CharType dtype) { - if (!dry_run) - { - write_bjdata_ndarray_values(array_data, dtype); - return true; - } - for (const auto& el : array_data) { bool ok = true; @@ -4311,43 +4279,39 @@ class binary_writer { case 'U': case 'B': - ok = write_bjdata_ndarray_element(el, dry_run); + ok = bjdata_ndarray_value_in_range(el); break; case 'C': // BJData limits a char to 0..127 (the intersection of the int8 and uint8 ranges) ok = bjdata_ndarray_value_in_range(el) && bjdata_ndarray_value_in_range(el); break; case 'i': - ok = write_bjdata_ndarray_element(el, dry_run); + ok = bjdata_ndarray_value_in_range(el); break; case 'u': - ok = write_bjdata_ndarray_element(el, dry_run); + ok = bjdata_ndarray_value_in_range(el); break; case 'I': - ok = write_bjdata_ndarray_element(el, dry_run); + ok = bjdata_ndarray_value_in_range(el); break; case 'm': - ok = write_bjdata_ndarray_element(el, dry_run); + ok = bjdata_ndarray_value_in_range(el); break; case 'l': - ok = write_bjdata_ndarray_element(el, dry_run); + ok = bjdata_ndarray_value_in_range(el); break; case 'M': - ok = write_bjdata_ndarray_element(el, dry_run); + ok = bjdata_ndarray_value_in_range(el); break; case 'L': - ok = write_bjdata_ndarray_element(el, dry_run); + ok = bjdata_ndarray_value_in_range(el); break; case 'd': - ok = write_bjdata_ndarray_float_element(el, dry_run); + ok = bjdata_ndarray_float_in_range(el); break; case 'D': default: // 'D' (double) already spans the full range of number_float_t - if (!dry_run) - { - write_number(el.template get(), true); - } break; } if (!ok) @@ -4434,7 +4398,7 @@ class binary_writer const auto dim = el.template get(); if (!value_in_range_of(dim)) { - return true; + return true; // LCOV_EXCL_LINE (size_t is narrower than 64 bits) } const auto dim_size = static_cast(dim); @@ -4484,7 +4448,7 @@ class binary_writer // wrap (integers) or overflow to infinity (the "single" precision // float) instead of being reported, so such an object falls back to // a plain object encoding as well - if (!write_bjdata_ndarray_elements(array_data, dtype, true)) + if (!bjdata_ndarray_elements_in_range(array_data, dtype)) { return true; } @@ -4496,7 +4460,7 @@ class binary_writer write_ubjson(array_size, use_count, use_type, true, true, bjdata_version); - write_bjdata_ndarray_elements(array_data, dtype, false); + write_bjdata_ndarray_values(array_data, dtype); return false; } @@ -4822,8 +4786,8 @@ class binary_writer run are encoded into space reserved from the sink, as in @ref write_array_elements; an integer that BON8 cannot write (an unsigned one above the int64 range, or one of a number type wider than 64 bits) throws - from write_bon8_value, like any other value that is not a number. Kept out - of line, like @ref write_cbor_array. + out_of_range.407, as write_bon8_value does. Kept out of line, like + @ref write_cbor_array. */ template JSON_HEDLEY_NEVER_INLINE @@ -4873,10 +4837,10 @@ class binary_writer string_open = false; } - if (unencodable) + if (JSON_HEDLEY_UNLIKELY(unencodable)) { - write_bon8_value(*it, string_open, depth + 1); - ++it; + // an integer beyond the int64 range, as write_bon8_value would report it + throw_integer_out_of_range(*it, input_format_t::bon8, "int64"); } } } @@ -4911,14 +4875,13 @@ class binary_writer break; case value_t::number_integer: - if (JSON_HEDLEY_LIKELY(integer_fits(value.m_data.m_value.number_integer))) + if (JSON_HEDLEY_UNLIKELY(!integer_fits(value.m_data.m_value.number_integer))) { - write_bon8_integer(static_cast(value.m_data.m_value.number_integer)); - string_open = false; - break; + // beyond 64 bits (number types wider than that) + throw_integer_out_of_range(value, input_format_t::bon8, "int64"); } - // beyond 64 bits (number types wider than that): throws - write_bon8_value(value, string_open, depth + 1); + write_bon8_integer(static_cast(value.m_data.m_value.number_integer)); + string_open = false; break; case value_t::number_float: @@ -4927,14 +4890,13 @@ class binary_writer break; case value_t::number_unsigned: - if (JSON_HEDLEY_LIKELY(integer_fits_max(value.m_data.m_value.number_unsigned))) + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(value.m_data.m_value.number_unsigned))) { - write_bon8_integer(static_cast(value.m_data.m_value.number_unsigned)); - string_open = false; - break; + // too large for BON8 + throw_integer_out_of_range(value, input_format_t::bon8, "int64"); } - // too large for BON8: throws - write_bon8_value(value, string_open, depth + 1); + write_bon8_integer(static_cast(value.m_data.m_value.number_unsigned)); + string_open = false; break; case value_t::object: diff --git a/include/nlohmann/detail/output/serializer.hpp b/include/nlohmann/detail/output/serializer.hpp index 4656dbdf1..fc8e2b384 100644 --- a/include/nlohmann/detail/output/serializer.hpp +++ b/include/nlohmann/detail/output/serializer.hpp @@ -1117,7 +1117,7 @@ class serializer // character if (EnsureAscii) { - JSON_ASSERT(codepoint >= 0x7F); + JSON_ASSERT(codepoint == 0x7F); if (error_handler == error_handler_t::keep) { // this character was buffered as raw bytes below @@ -1128,21 +1128,13 @@ class serializer // character instead bytes = bytes_after_last_accept; } - if (codepoint <= 0xFFFF) - { - write_u_escape(bytes, static_cast(codepoint)); - } - else - { - write_u_escape(bytes, static_cast(0xD7C0u + (codepoint >> 10u))); - write_u_escape(bytes, static_cast(0xDC00u + (codepoint & 0x3FFu))); - } + write_u_escape(bytes, static_cast(codepoint)); } else { // copy byte to buffer (all previous bytes of the // sequence have been copied in the default case below) - string_buffer[bytes++] = s[i]; + string_buffer[bytes++] = s[i]; // LCOV_EXCL_LINE (well-formed sequences are copied in runs) } reserve_string_buffer(bytes); diff --git a/include/nlohmann/detail/string_concat.hpp b/include/nlohmann/detail/string_concat.hpp index 74002c878..d7ad45af3 100644 --- a/include/nlohmann/detail/string_concat.hpp +++ b/include/nlohmann/detail/string_concat.hpp @@ -139,7 +139,7 @@ inline OutStringType concat(Args && ... args) str.reserve(concat_length(args...)); concat_into(str, std::forward(args)...); return str; -} +} // LCOV_EXCL_LINE (exception cleanup only) } // namespace detail NLOHMANN_JSON_NAMESPACE_END diff --git a/include/nlohmann/detail/string_escape.hpp b/include/nlohmann/detail/string_escape.hpp index 1a4b6c501..3e6dc13ae 100644 --- a/include/nlohmann/detail/string_escape.hpp +++ b/include/nlohmann/detail/string_escape.hpp @@ -83,10 +83,6 @@ template inline void unescape(StringType& s) { auto pos = s.find_first_of('~', 0); - if (pos == StringType::npos) - { - return; - } StringType result; result.reserve(s.size()); diff --git a/include/nlohmann/detail/string_utils.hpp b/include/nlohmann/detail/string_utils.hpp index 9b44623a5..c7ec18b3d 100644 --- a/include/nlohmann/detail/string_utils.hpp +++ b/include/nlohmann/detail/string_utils.hpp @@ -36,7 +36,7 @@ StringType to_string(std::size_t value) StringType result; int_to_string(result, value); return result; -} +} // LCOV_EXCL_LINE (exception cleanup only) /// @return a byte as two uppercase hexadecimal digits inline std::string hex_byte(const std::uint8_t byte) @@ -46,7 +46,7 @@ inline std::string hex_byte(const std::uint8_t byte) result[0] = nibble_to_hex[byte / 16]; result[1] = nibble_to_hex[byte % 16]; return result; -} +} // LCOV_EXCL_LINE (exception cleanup only) /*! @brief write a UTF-16 code unit as four lowercase hexadecimal digits @@ -390,7 +390,7 @@ inline StringType sanitize_utf8(const StringType& s, const error_handler_t error } return result; -} +} // LCOV_EXCL_LINE (exception cleanup only) } // namespace detail NLOHMANN_JSON_NAMESPACE_END diff --git a/include/nlohmann/detail/view/compare.hpp b/include/nlohmann/detail/view/compare.hpp index 96a14efbb..239871f01 100644 --- a/include/nlohmann/detail/view/compare.hpp +++ b/include/nlohmann/detail/view/compare.hpp @@ -428,28 +428,19 @@ class view_side return string_view_t(m_view.m_doc->str(*m_view.m_node), m_view.m_node->len); } - /// a number, boolean, or null as a basic_json value (no allocation) + /// a number as a basic_json value (no allocation); only called for numbers BasicJsonType scalar() const { - switch (m_view.type()) + const value_t t = m_view.type(); + if (t == value_t::number_integer) { - case value_t::number_integer: - return BasicJsonType(m_view.template get()); - case value_t::number_unsigned: - return BasicJsonType(m_view.template get()); - case value_t::number_float: - return BasicJsonType(m_view.template get()); - case value_t::boolean: - return BasicJsonType(m_view.template get()); - case value_t::null: - case value_t::object: - case value_t::array: - case value_t::string: - case value_t::binary: - case value_t::discarded: - default: - return BasicJsonType(nullptr); + return BasicJsonType(m_view.template get()); } + if (t == value_t::number_unsigned) + { + return BasicJsonType(m_view.template get()); + } + return BasicJsonType(m_view.template get()); } /// start at the first element of this array diff --git a/include/nlohmann/detail/view/edit.hpp b/include/nlohmann/detail/view/edit.hpp index b572b4848..8ddf3f849 100644 --- a/include/nlohmann/detail/view/edit.hpp +++ b/include/nlohmann/detail/view/edit.hpp @@ -96,11 +96,11 @@ NLOHMANN_VIEW_NOINLINE inline void check_utf8_scan(const unsigned char* p, std:: } if (*q >= 0x80) { - check_utf8_bytewise(p, n, static_cast(q - p)); - return; // not reached: the scan stopped at ill-formed UTF-8 + break; // ill-formed UTF-8: reported below } ++q; // a quote, backslash, or control character } + check_utf8_bytewise(p, n, static_cast(q - p)); } inline void check_utf8(const char* s, std::size_t n) diff --git a/include/nlohmann/detail/view/edit_storage.hpp b/include/nlohmann/detail/view/edit_storage.hpp index e5b7ddb90..10898f99f 100644 --- a/include/nlohmann/detail/view/edit_storage.hpp +++ b/include/nlohmann/detail/view/edit_storage.hpp @@ -243,7 +243,7 @@ inline node* find_parent(const document_data& d, const node* target) } if (target == lo) { - return nullptr; + return nullptr; // LCOV_EXCL_LINE (the root, or a new value's root, which is linked) } for (;;) { diff --git a/include/nlohmann/detail/view/errors.hpp b/include/nlohmann/detail/view/errors.hpp index 64c514738..de35d6a51 100644 --- a/include/nlohmann/detail/view/errors.hpp +++ b/include/nlohmann/detail/view/errors.hpp @@ -419,10 +419,12 @@ struct throwing_sax return true; } + // LCOV_EXCL_START (throw_parse_failure parses JSON only) bool binary(binary_t& /*unused*/) { return true; } + // LCOV_EXCL_STOP bool start_object(std::size_t /*unused*/) { diff --git a/include/nlohmann/detail/view/materialize.hpp b/include/nlohmann/detail/view/materialize.hpp index 1dcd1940c..71b29a13c 100644 --- a/include/nlohmann/detail/view/materialize.hpp +++ b/include/nlohmann/detail/view/materialize.hpp @@ -65,14 +65,9 @@ struct position_source return token{end - start}; } - /// the range [first, last), or none if either is npos + /// the range [first, last) void set(std::size_t first, std::size_t last) noexcept { - if (first == std::string::npos || last == std::string::npos) - { - clear(); - return; - } start = first; end = last; } @@ -191,11 +186,11 @@ struct position_source first = n.off; last = n.off + 4u; break; - case node_kind::object: + case node_kind::object: // LCOV_EXCL_LINE case node_kind::array: case node_kind::link: default: - break; + break; // LCOV_EXCL_LINE } // (a crafted image can point past the text) if (last == std::string::npos || last > d.size) diff --git a/include/nlohmann/detail/view/serializer.hpp b/include/nlohmann/detail/view/serializer.hpp index 28e1d647a..44e5a2861 100644 --- a/include/nlohmann/detail/view/serializer.hpp +++ b/include/nlohmann/detail/view/serializer.hpp @@ -1394,20 +1394,10 @@ class view_serializer /// a float node as dump() writes it void write_float_node(const node& n) { - write_float_node(n, fast_double {}); - } - - void write_float_node(const node& n, std::false_type /*other*/) - { + // (only for types other than double: doubles go through write_double_ahead()) write_float(float_value(m_doc, n)); } - void write_float_node(const node& n, std::true_type /*double*/) - { - m_out.reserve(64); - m_out.set_cursor(write_double_at(m_out.cursor(), n)); - } - /*! @brief (doubles) the float at n as dump() writes it, at w (64 bytes of room) @@ -1502,7 +1492,7 @@ class view_serializer { const string_t dumped = BasicJsonType(string_t(reinterpret_cast(s), n)).dump(); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) static_cast(dumped); - } + } // LCOV_EXCL_LINE (dump() throws: the callers pass only ill-formed UTF-8) template void write_codepoint(std::uint32_t codepoint, const unsigned char* bytes, std::size_t len) @@ -1535,15 +1525,7 @@ class view_serializer } if (codepoint <= 0x1F || (EnsureAscii && codepoint >= 0x7F)) { - if (codepoint <= 0xFFFF) - { - write_u_escape(codepoint); - } - else - { - write_u_escape(0xD7C0u + (codepoint >> 10u)); - write_u_escape(0xDC00u + (codepoint & 0x3FFu)); - } + write_u_escape(codepoint); // (a single byte: the only caller passes s[i]) return; } m_out.put(reinterpret_cast(bytes), len); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) LCOV_EXCL_LINE (printable characters are copied in runs) diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 28cad76ba..0447f8823 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -400,7 +400,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec result["compiler"] = {{"family", "clang"}, {"version", __clang_version__}}; #elif defined(__GNUC__) || defined(__GNUG__) result["compiler"] = {{"family", "gcc"}, {"version", detail::concat( - detail::to_string(static_cast(__GNUC__)), '.', + detail::to_string(static_cast(__GNUC__)), '.', // LCOV_EXCL_LINE detail::to_string(static_cast(__GNUC_MINOR__)), '.', detail::to_string(static_cast(__GNUC_PATCHLEVEL__))) } @@ -676,14 +676,14 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec case value_t::binary: binary = create(); break; - case value_t::null: + case value_t::null: // LCOV_EXCL_LINE case value_t::boolean: case value_t::number_integer: case value_t::number_unsigned: case value_t::number_float: case value_t::discarded: default: - break; + break; // LCOV_EXCL_LINE } } @@ -1037,14 +1037,14 @@ public: destroy_container(t); break; - case value_t::null: + case value_t::null: // LCOV_EXCL_LINE case value_t::boolean: case value_t::number_integer: case value_t::number_unsigned: case value_t::number_float: case value_t::discarded: default: - break; + break; // LCOV_EXCL_LINE } } }; @@ -1217,9 +1217,9 @@ public: return is_array() ? array_capacity() : object_capacity(); } #else - static constexpr std::size_t container_capacity() noexcept + static constexpr std::size_t container_capacity() noexcept // LCOV_EXCL_LINE { - return detail::unknown_size(); + return detail::unknown_size(); // LCOV_EXCL_LINE } #endif @@ -1405,7 +1405,7 @@ public: /// into this switch is worth a few percent when copying a value made /// mostly of numbers JSON_HEDLEY_ALWAYS_INLINE - static void copy_leaf_value(const basic_json& src, basic_json& dst) + static void copy_leaf_value(const basic_json& src, basic_json& dst) // LCOV_EXCL_LINE { switch (src.m_data.m_type) { @@ -5589,7 +5589,7 @@ public: /// their inlined code at each call site small. Binary values are /// rare and left to equal_general(). JSON_HEDLEY_ALWAYS_INLINE - static int equal_leaf(const_reference lhs, const_reference rhs) noexcept + static int equal_leaf(const_reference lhs, const_reference rhs) noexcept // LCOV_EXCL_LINE { #ifdef __GNUC__ JSON_HEDLEY_DIAGNOSTIC_PUSH @@ -6898,7 +6898,7 @@ public: basic_json result(value_t::object); json_pointer::flatten("", *this, result); return result; - } + } // LCOV_EXCL_LINE (exception cleanup only) /// @brief unflatten a previously flattened JSON value /// @sa https://json.nlohmann.me/api/basic_json/unflatten/ @@ -7611,7 +7611,7 @@ public: result.append(path.data(), path.size()); diff_append_key(result, key); return result; - } + } // LCOV_EXCL_LINE (exception cleanup only) /// @brief the path of the element @a index of the value at @a path static string_t diff_index_path(const string_t& path, const std::size_t index) @@ -7621,7 +7621,7 @@ public: result.append(path.data(), path.size()); diff_append_index(result, index); return result; - } + } // LCOV_EXCL_LINE (exception cleanup only) /// @brief append the "remove" operations for the elements of array /// @a source from @a index on, and the "add" operations for the @@ -7781,7 +7781,7 @@ public: // operations if (source == target) { - return false; + return false; // LCOV_EXCL_LINE (equal objects with the same members in the same order return early) } // slow path: the common keys are in a different relative diff --git a/include/nlohmann/json_view.hpp b/include/nlohmann/json_view.hpp index 043d09036..64aa8f723 100644 --- a/include/nlohmann/json_view.hpp +++ b/include/nlohmann/json_view.hpp @@ -1705,9 +1705,11 @@ class basic_json_document { // (as before: the library's adapter decides what a null stream yields) detail::view::append_adapter(buf, detail::input_adapter(file)); - return; } - detail::view::append_file(buf, file); + else + { + detail::view::append_file(buf, file); + } } #endif diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 7029ed34e..1818721c1 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -3637,10 +3637,6 @@ template inline void unescape(StringType& s) { auto pos = s.find_first_of('~', 0); - if (pos == StringType::npos) - { - return; - } StringType result; result.reserve(s.size()); @@ -5516,7 +5512,7 @@ inline OutStringType concat(Args && ... args) str.reserve(concat_length(args...)); concat_into(str, std::forward(args)...); return str; -} +} // LCOV_EXCL_LINE (exception cleanup only) } // namespace detail NLOHMANN_JSON_NAMESPACE_END @@ -6238,7 +6234,7 @@ const typename BasicJsonType::binary_t& binary_value(const BasicJsonType& j) } template::value, int> = 0> -typename View::basic_json_type::binary_t binary_value(const View& /*unused*/) +typename View::basic_json_type::binary_t binary_value(const View& /*unused*/) // LCOV_EXCL_LINE { return {}; // LCOV_EXCL_LINE } @@ -7164,7 +7160,7 @@ StringType to_string(std::size_t value) StringType result; int_to_string(result, value); return result; -} +} // LCOV_EXCL_LINE (exception cleanup only) /// @return a byte as two uppercase hexadecimal digits inline std::string hex_byte(const std::uint8_t byte) @@ -7174,7 +7170,7 @@ inline std::string hex_byte(const std::uint8_t byte) result[0] = nibble_to_hex[byte / 16]; result[1] = nibble_to_hex[byte % 16]; return result; -} +} // LCOV_EXCL_LINE (exception cleanup only) /*! @brief write a UTF-16 code unit as four lowercase hexadecimal digits @@ -7518,7 +7514,7 @@ inline StringType sanitize_utf8(const StringType& s, const error_handler_t error } return result; -} +} // LCOV_EXCL_LINE (exception cleanup only) } // namespace detail NLOHMANN_JSON_NAMESPACE_END @@ -9685,7 +9681,7 @@ class input_stream_adapter } else { - current_in_get_area = true; + current_in_get_area = true; // LCOV_EXCL_LINE (called only when the get area is exhausted) } #endif return res; @@ -13477,11 +13473,10 @@ class lexer : public lexer_base template char_int_type get_after_string_run(std::true_type /*bulk*/, std::integral_constant keep) { - // a pending unget must be consumed through the normal path first - if (next_unget) - { - return get(); - } + // scan_string() is entered from scan_after_whitespace(), whose current + // was read by skip_whitespace() (its first get() clears a pending + // unget), and nothing in scan_string() calls unget() + JSON_ASSERT(!next_unget); const std::size_t remaining = ia.bulk_remaining(); if (remaining == 0) { @@ -14556,11 +14551,9 @@ scan_number_done: */ token_type scan_number_bulk_contiguous() { - // a pending unget offsets the buffer position from current; fall back - if (next_unget) - { - return token_type::uninitialized; - } + // called right after scan_after_whitespace() read current (see + // get_after_string_run()), so no unget is pending + JSON_ASSERT(!next_unget); const std::size_t rem = ia.bulk_remaining(); if (rem == 0) { @@ -14581,11 +14574,7 @@ scan_number_done: if (data[0] == '-') { number_type = token_type::value_integer; - i = 1; - if (i >= avail) - { - return token_type::uninitialized; - } + i = 1; // avail >= 2: rem == 0 returned above } const std::size_t integer_begin = i; if (data[i] == '0') @@ -16646,7 +16635,7 @@ class json_sax_dom_parser : public json_sax_dom_error_state ordered_data.members[f.ordered.dest_member].second = std::move(value); } - static void place_member(const frame& /*f*/, BasicJsonType&& /*value*/, std::false_type /*indexed*/) noexcept {} + static void place_member(const frame& /*f*/, BasicJsonType&& /*value*/, std::false_type /*indexed*/) noexcept {} // LCOV_EXCL_LINE /// store the members of a new object directly in it bool start_stored_object(std::size_t len) @@ -23813,7 +23802,7 @@ class parser } return error_msg; - } + } // LCOV_EXCL_LINE (exception cleanup only) private: /// callback function @@ -25124,7 +25113,7 @@ class json_pointer result.reference_tokens.insert(result.reference_tokens.end(), lhs.reference_tokens.begin(), lhs.reference_tokens.end()); result.reference_tokens.insert(result.reference_tokens.end(), rhs.reference_tokens.begin(), rhs.reference_tokens.end()); return result; - } + } // LCOV_EXCL_LINE (exception cleanup only) /// @brief create a new JSON pointer by appending the unescaped token at the end of the JSON pointer /// @sa https://json.nlohmann.me/api/json_pointer/operator_slash/ @@ -25135,7 +25124,7 @@ class json_pointer result.reference_tokens.insert(result.reference_tokens.end(), lhs.reference_tokens.begin(), lhs.reference_tokens.end()); result.reference_tokens.push_back(std::move(token)); return result; - } + } // LCOV_EXCL_LINE (exception cleanup only) /// @brief create a new JSON pointer by appending the array-index-token at the end of the JSON pointer /// @sa https://json.nlohmann.me/api/json_pointer/operator_slash/ @@ -27000,7 +26989,7 @@ class json_pointer { // create an entry in the array current = ¤t->operator[](array_index(token)); - break; + break; // LCOV_EXCL_LINE (needs an array index with 10 or more digits: more than 1e9 elements) } /* @@ -27037,7 +27026,7 @@ class json_pointer json_pointer result; result.reference_tokens = reference_tokens; return result; - } + } // LCOV_EXCL_LINE (exception cleanup only) json_pointer convert()&& { @@ -28807,7 +28796,7 @@ class binary_writer while (n != 0); std::reverse(result.begin(), result.end()); return result; - } + } // LCOV_EXCL_LINE (exception cleanup only) /*! @brief the decimal digits of the integer @a n, with a leading '-' if it @@ -28834,7 +28823,7 @@ class binary_writer } std::reverse(result.begin(), result.end()); return result; - } + } // LCOV_EXCL_LINE (exception cleanup only) /*! @brief throws because the integer @a j is too large for @a format @@ -29766,7 +29755,7 @@ class binary_writer std::memcpy(out, &n, sizeof(NumberType)); if (!is_little_endian) { - std::reverse(out, out + sizeof(NumberType)); + std::reverse(out, out + sizeof(NumberType)); // LCOV_EXCL_LINE (big-endian hosts only) } } @@ -29895,7 +29884,7 @@ class binary_writer const string_t& value = *j.m_data.m_value.string; if (JSON_HEDLEY_UNLIKELY(!value_in_range_of(value.size() + 1ul))) { - return false; + return false; // LCOV_EXCL_LINE (a string of 2 GiB or more) } out[0] = to_char_type(0x02); store_little_endian(p, static_cast(value.size() + 1ul)); @@ -30352,7 +30341,7 @@ class binary_writer const std::size_t size = oa.written() - pos; if (JSON_HEDLEY_UNLIKELY(!value_in_range_of(size))) { - return false; + return false; // LCOV_EXCL_LINE (a document of 2 GiB or more) } const auto le_size = static_cast(size); oa.overwrite(pos, is_little_endian ? le_size : byteswap(le_size)); @@ -30480,7 +30469,7 @@ class binary_writer { if (JSON_HEDLEY_UNLIKELY(!close_bson_container(nested_size_pos))) { - return false; + return false; // LCOV_EXCL_LINE (a document of 2 GiB or more) } continue; } @@ -30495,7 +30484,7 @@ class binary_writer if (JSON_HEDLEY_UNLIKELY(!close_bson_container(current.size_slot))) { - return false; + return false; // LCOV_EXCL_LINE (a document of 2 GiB or more) } if (parents.empty()) { @@ -30752,10 +30741,10 @@ class binary_writer Selected through masks rather than branches, as the type and the sign of the values in an array of mixed signs alternate unpredictably (non-negative - integers are number_unsigned after parsing). Up to 64 bits, the sign is bit - 63 of the result (a narrower integer is sign-extended); computing it with - masks keeps GCC from branching on the type. A wider integer has to be - compared. + integers are number_unsigned after parsing). The sign is bit 63 of the + result (a narrower integer is sign-extended); computing it with masks keeps + GCC from branching on the type. Only called if @ref integers_fit_64_bits(); + a wider number type is written with range checks on the whole value. */ static std::uint64_t integer_bits(const BasicJsonType& j, std::uint64_t& sign_mask) noexcept { @@ -30763,9 +30752,7 @@ class binary_writer const std::uint64_t raw = is_signed ? static_cast(j.m_data.m_value.number_integer) : static_cast(j.m_data.m_value.number_unsigned); - sign_mask = sizeof(typename BasicJsonType::number_integer_t) <= sizeof(std::uint64_t) - ? (std::uint64_t{0} - static_cast(is_signed)) & (std::uint64_t{0} - (raw >> 63u)) - : std::uint64_t{0} - static_cast(is_signed && j.m_data.m_value.number_integer < 0); + sign_mask = (std::uint64_t{0} - static_cast(is_signed)) & (std::uint64_t{0} - (raw >> 63u)); return raw; } @@ -32132,57 +32119,34 @@ class binary_writer } /*! - @brief validate (dry_run) or write one BJData ND-array element of integer dtype @a T - @return whether @a el's value is in range of @a T; always true when @a dry_run is false - */ - template - bool write_bjdata_ndarray_element(const BasicJsonType& el, const bool dry_run) - { - if (dry_run) - { - return bjdata_ndarray_value_in_range(el); - } - using storage_type = typename std::conditional::value, std::uint64_t, std::int64_t>::type; - write_number(static_cast(el.template get()), true); - return true; - } - - /*! - @brief validate (dry_run) or write one BJData ND-array element of dtype 'd' (single precision) + @brief whether the BJData ND-array element @a el can be written with dtype 'd' (single precision) @return whether @a el's value survives narrowing to float and back without any change - (so the ND-array round-trips exactly), or is infinite or NaN; always true when - @a dry_run is false + (so the ND-array round-trips exactly), or is infinite or NaN */ - bool write_bjdata_ndarray_float_element(const BasicJsonType& el, const bool dry_run) + static bool bjdata_ndarray_float_in_range(const BasicJsonType& el) noexcept { const auto dval = static_cast(el.m_data.m_value.number_float); - if (dry_run) - { #ifdef __GNUC__ - JSON_HEDLEY_DIAGNOSTIC_PUSH - JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") + JSON_HEDLEY_DIAGNOSTIC_PUSH + JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") #endif - // a value that would be rounded (rather than exactly represented) by the - // narrowing to float is treated like an out-of-range integer element; this - // is the same criterion write_compact_float() uses for CBOR/MessagePack - const bool in_range = std::isnan(dval) || - (dval >= static_cast(std::numeric_limits::lowest()) && - dval <= static_cast((std::numeric_limits::max)()) && - static_cast(static_cast(dval)) == dval) || - std::isinf(dval); + // a value that would be rounded (rather than exactly represented) by the + // narrowing to float is treated like an out-of-range integer element; this + // is the same criterion write_compact_float() uses for CBOR/MessagePack + return std::isnan(dval) || + (dval >= static_cast(std::numeric_limits::lowest()) && + dval <= static_cast((std::numeric_limits::max)()) && + static_cast(static_cast(dval)) == dval) || + std::isinf(dval); #ifdef __GNUC__ - JSON_HEDLEY_DIAGNOSTIC_POP + JSON_HEDLEY_DIAGNOSTIC_POP #endif - return in_range; - } - write_number(static_cast(dval), true); - return true; } /*! @brief write the elements of an ND-array as the number kind @a dtype names (see @ref write_bjdata_ndarray), after they have been found to be - in range by a dry run of @ref write_bjdata_ndarray_elements + in range by @ref bjdata_ndarray_elements_in_range The elements are encoded in blocks into space reserved from the sink. */ @@ -32264,20 +32228,13 @@ class binary_writer } /*! - @brief validate or write every element of a BJData ND-array's `_ArrayData_` + @brief whether every element of a BJData ND-array's `_ArrayData_` is in range for its dtype @param[in] array_data the `_ArrayData_` array @param[in] dtype the ND-array dtype marker, as returned by bjdata_ndarray_type_marker() - @param[in] dry_run true to only range-check each element, false to write it - @return whether every element is in range for @a dtype (always true when @a dry_run is false) + @return whether every element is in range for @a dtype */ - bool write_bjdata_ndarray_elements(const BasicJsonType& array_data, const CharType dtype, const bool dry_run) + static bool bjdata_ndarray_elements_in_range(const BasicJsonType& array_data, const CharType dtype) { - if (!dry_run) - { - write_bjdata_ndarray_values(array_data, dtype); - return true; - } - for (const auto& el : array_data) { bool ok = true; @@ -32285,43 +32242,39 @@ class binary_writer { case 'U': case 'B': - ok = write_bjdata_ndarray_element(el, dry_run); + ok = bjdata_ndarray_value_in_range(el); break; case 'C': // BJData limits a char to 0..127 (the intersection of the int8 and uint8 ranges) ok = bjdata_ndarray_value_in_range(el) && bjdata_ndarray_value_in_range(el); break; case 'i': - ok = write_bjdata_ndarray_element(el, dry_run); + ok = bjdata_ndarray_value_in_range(el); break; case 'u': - ok = write_bjdata_ndarray_element(el, dry_run); + ok = bjdata_ndarray_value_in_range(el); break; case 'I': - ok = write_bjdata_ndarray_element(el, dry_run); + ok = bjdata_ndarray_value_in_range(el); break; case 'm': - ok = write_bjdata_ndarray_element(el, dry_run); + ok = bjdata_ndarray_value_in_range(el); break; case 'l': - ok = write_bjdata_ndarray_element(el, dry_run); + ok = bjdata_ndarray_value_in_range(el); break; case 'M': - ok = write_bjdata_ndarray_element(el, dry_run); + ok = bjdata_ndarray_value_in_range(el); break; case 'L': - ok = write_bjdata_ndarray_element(el, dry_run); + ok = bjdata_ndarray_value_in_range(el); break; case 'd': - ok = write_bjdata_ndarray_float_element(el, dry_run); + ok = bjdata_ndarray_float_in_range(el); break; case 'D': default: // 'D' (double) already spans the full range of number_float_t - if (!dry_run) - { - write_number(el.template get(), true); - } break; } if (!ok) @@ -32408,7 +32361,7 @@ class binary_writer const auto dim = el.template get(); if (!value_in_range_of(dim)) { - return true; + return true; // LCOV_EXCL_LINE (size_t is narrower than 64 bits) } const auto dim_size = static_cast(dim); @@ -32458,7 +32411,7 @@ class binary_writer // wrap (integers) or overflow to infinity (the "single" precision // float) instead of being reported, so such an object falls back to // a plain object encoding as well - if (!write_bjdata_ndarray_elements(array_data, dtype, true)) + if (!bjdata_ndarray_elements_in_range(array_data, dtype)) { return true; } @@ -32470,7 +32423,7 @@ class binary_writer write_ubjson(array_size, use_count, use_type, true, true, bjdata_version); - write_bjdata_ndarray_elements(array_data, dtype, false); + write_bjdata_ndarray_values(array_data, dtype); return false; } @@ -32796,8 +32749,8 @@ class binary_writer run are encoded into space reserved from the sink, as in @ref write_array_elements; an integer that BON8 cannot write (an unsigned one above the int64 range, or one of a number type wider than 64 bits) throws - from write_bon8_value, like any other value that is not a number. Kept out - of line, like @ref write_cbor_array. + out_of_range.407, as write_bon8_value does. Kept out of line, like + @ref write_cbor_array. */ template JSON_HEDLEY_NEVER_INLINE @@ -32847,10 +32800,10 @@ class binary_writer string_open = false; } - if (unencodable) + if (JSON_HEDLEY_UNLIKELY(unencodable)) { - write_bon8_value(*it, string_open, depth + 1); - ++it; + // an integer beyond the int64 range, as write_bon8_value would report it + throw_integer_out_of_range(*it, input_format_t::bon8, "int64"); } } } @@ -32885,14 +32838,13 @@ class binary_writer break; case value_t::number_integer: - if (JSON_HEDLEY_LIKELY(integer_fits(value.m_data.m_value.number_integer))) + if (JSON_HEDLEY_UNLIKELY(!integer_fits(value.m_data.m_value.number_integer))) { - write_bon8_integer(static_cast(value.m_data.m_value.number_integer)); - string_open = false; - break; + // beyond 64 bits (number types wider than that) + throw_integer_out_of_range(value, input_format_t::bon8, "int64"); } - // beyond 64 bits (number types wider than that): throws - write_bon8_value(value, string_open, depth + 1); + write_bon8_integer(static_cast(value.m_data.m_value.number_integer)); + string_open = false; break; case value_t::number_float: @@ -32901,14 +32853,13 @@ class binary_writer break; case value_t::number_unsigned: - if (JSON_HEDLEY_LIKELY(integer_fits_max(value.m_data.m_value.number_unsigned))) + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(value.m_data.m_value.number_unsigned))) { - write_bon8_integer(static_cast(value.m_data.m_value.number_unsigned)); - string_open = false; - break; + // too large for BON8 + throw_integer_out_of_range(value, input_format_t::bon8, "int64"); } - // too large for BON8: throws - write_bon8_value(value, string_open, depth + 1); + write_bon8_integer(static_cast(value.m_data.m_value.number_unsigned)); + string_open = false; break; case value_t::object: @@ -36365,7 +36316,7 @@ class serializer // character if (EnsureAscii) { - JSON_ASSERT(codepoint >= 0x7F); + JSON_ASSERT(codepoint == 0x7F); if (error_handler == error_handler_t::keep) { // this character was buffered as raw bytes below @@ -36376,21 +36327,13 @@ class serializer // character instead bytes = bytes_after_last_accept; } - if (codepoint <= 0xFFFF) - { - write_u_escape(bytes, static_cast(codepoint)); - } - else - { - write_u_escape(bytes, static_cast(0xD7C0u + (codepoint >> 10u))); - write_u_escape(bytes, static_cast(0xDC00u + (codepoint & 0x3FFu))); - } + write_u_escape(bytes, static_cast(codepoint)); } else { // copy byte to buffer (all previous bytes of the // sequence have been copied in the default case below) - string_buffer[bytes++] = s[i]; + string_buffer[bytes++] = s[i]; // LCOV_EXCL_LINE (well-formed sequences are copied in runs) } reserve_string_buffer(bytes); @@ -38231,7 +38174,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec result["compiler"] = {{"family", "clang"}, {"version", __clang_version__}}; #elif defined(__GNUC__) || defined(__GNUG__) result["compiler"] = {{"family", "gcc"}, {"version", detail::concat( - detail::to_string(static_cast(__GNUC__)), '.', + detail::to_string(static_cast(__GNUC__)), '.', // LCOV_EXCL_LINE detail::to_string(static_cast(__GNUC_MINOR__)), '.', detail::to_string(static_cast(__GNUC_PATCHLEVEL__))) } @@ -38507,14 +38450,14 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec case value_t::binary: binary = create(); break; - case value_t::null: + case value_t::null: // LCOV_EXCL_LINE case value_t::boolean: case value_t::number_integer: case value_t::number_unsigned: case value_t::number_float: case value_t::discarded: default: - break; + break; // LCOV_EXCL_LINE } } @@ -38868,14 +38811,14 @@ public: destroy_container(t); break; - case value_t::null: + case value_t::null: // LCOV_EXCL_LINE case value_t::boolean: case value_t::number_integer: case value_t::number_unsigned: case value_t::number_float: case value_t::discarded: default: - break; + break; // LCOV_EXCL_LINE } } }; @@ -39048,9 +38991,9 @@ public: return is_array() ? array_capacity() : object_capacity(); } #else - static constexpr std::size_t container_capacity() noexcept + static constexpr std::size_t container_capacity() noexcept // LCOV_EXCL_LINE { - return detail::unknown_size(); + return detail::unknown_size(); // LCOV_EXCL_LINE } #endif @@ -39236,7 +39179,7 @@ public: /// into this switch is worth a few percent when copying a value made /// mostly of numbers JSON_HEDLEY_ALWAYS_INLINE - static void copy_leaf_value(const basic_json& src, basic_json& dst) + static void copy_leaf_value(const basic_json& src, basic_json& dst) // LCOV_EXCL_LINE { switch (src.m_data.m_type) { @@ -43420,7 +43363,7 @@ public: /// their inlined code at each call site small. Binary values are /// rare and left to equal_general(). JSON_HEDLEY_ALWAYS_INLINE - static int equal_leaf(const_reference lhs, const_reference rhs) noexcept + static int equal_leaf(const_reference lhs, const_reference rhs) noexcept // LCOV_EXCL_LINE { #ifdef __GNUC__ JSON_HEDLEY_DIAGNOSTIC_PUSH @@ -44729,7 +44672,7 @@ public: basic_json result(value_t::object); json_pointer::flatten("", *this, result); return result; - } + } // LCOV_EXCL_LINE (exception cleanup only) /// @brief unflatten a previously flattened JSON value /// @sa https://json.nlohmann.me/api/basic_json/unflatten/ @@ -45442,7 +45385,7 @@ public: result.append(path.data(), path.size()); diff_append_key(result, key); return result; - } + } // LCOV_EXCL_LINE (exception cleanup only) /// @brief the path of the element @a index of the value at @a path static string_t diff_index_path(const string_t& path, const std::size_t index) @@ -45452,7 +45395,7 @@ public: result.append(path.data(), path.size()); diff_append_index(result, index); return result; - } + } // LCOV_EXCL_LINE (exception cleanup only) /// @brief append the "remove" operations for the elements of array /// @a source from @a index on, and the "add" operations for the @@ -45612,7 +45555,7 @@ public: // operations if (source == target) { - return false; + return false; // LCOV_EXCL_LINE (equal objects with the same members in the same order return early) } // slow path: the common keys are in a different relative diff --git a/single_include/nlohmann/json_view.hpp b/single_include/nlohmann/json_view.hpp index 0861bd95e..bf64662e2 100644 --- a/single_include/nlohmann/json_view.hpp +++ b/single_include/nlohmann/json_view.hpp @@ -3580,28 +3580,19 @@ class view_side return string_view_t(m_view.m_doc->str(*m_view.m_node), m_view.m_node->len); } - /// a number, boolean, or null as a basic_json value (no allocation) + /// a number as a basic_json value (no allocation); only called for numbers BasicJsonType scalar() const { - switch (m_view.type()) + const value_t t = m_view.type(); + if (t == value_t::number_integer) { - case value_t::number_integer: - return BasicJsonType(m_view.template get()); - case value_t::number_unsigned: - return BasicJsonType(m_view.template get()); - case value_t::number_float: - return BasicJsonType(m_view.template get()); - case value_t::boolean: - return BasicJsonType(m_view.template get()); - case value_t::null: - case value_t::object: - case value_t::array: - case value_t::string: - case value_t::binary: - case value_t::discarded: - default: - return BasicJsonType(nullptr); + return BasicJsonType(m_view.template get()); } + if (t == value_t::number_unsigned) + { + return BasicJsonType(m_view.template get()); + } + return BasicJsonType(m_view.template get()); } /// start at the first element of this array @@ -4478,14 +4469,9 @@ struct position_source return token{end - start}; } - /// the range [first, last), or none if either is npos + /// the range [first, last) void set(std::size_t first, std::size_t last) noexcept { - if (first == std::string::npos || last == std::string::npos) - { - clear(); - return; - } start = first; end = last; } @@ -4604,11 +4590,11 @@ struct position_source first = n.off; last = n.off + 4u; break; - case node_kind::object: + case node_kind::object: // LCOV_EXCL_LINE case node_kind::array: case node_kind::link: default: - break; + break; // LCOV_EXCL_LINE } // (a crafted image can point past the text) if (last == std::string::npos || last > d.size) @@ -5261,10 +5247,12 @@ struct throwing_sax return true; } + // LCOV_EXCL_START (throw_parse_failure parses JSON only) bool binary(binary_t& /*unused*/) { return true; } + // LCOV_EXCL_STOP bool start_object(std::size_t /*unused*/) { @@ -5572,7 +5560,7 @@ inline node* find_parent(const document_data& d, const node* target) } if (target == lo) { - return nullptr; + return nullptr; // LCOV_EXCL_LINE (the root, or a new value's root, which is linked) } for (;;) { @@ -6193,11 +6181,11 @@ NLOHMANN_VIEW_NOINLINE inline void check_utf8_scan(const unsigned char* p, std:: } if (*q >= 0x80) { - check_utf8_bytewise(p, n, static_cast(q - p)); - return; // not reached: the scan stopped at ill-formed UTF-8 + break; // ill-formed UTF-8: reported below } ++q; // a quote, backslash, or control character } + check_utf8_bytewise(p, n, static_cast(q - p)); } inline void check_utf8(const char* s, std::size_t n) @@ -9857,20 +9845,10 @@ class view_serializer /// a float node as dump() writes it void write_float_node(const node& n) { - write_float_node(n, fast_double {}); - } - - void write_float_node(const node& n, std::false_type /*other*/) - { + // (only for types other than double: doubles go through write_double_ahead()) write_float(float_value(m_doc, n)); } - void write_float_node(const node& n, std::true_type /*double*/) - { - m_out.reserve(64); - m_out.set_cursor(write_double_at(m_out.cursor(), n)); - } - /*! @brief (doubles) the float at n as dump() writes it, at w (64 bytes of room) @@ -9965,7 +9943,7 @@ class view_serializer { const string_t dumped = BasicJsonType(string_t(reinterpret_cast(s), n)).dump(); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) static_cast(dumped); - } + } // LCOV_EXCL_LINE (dump() throws: the callers pass only ill-formed UTF-8) template void write_codepoint(std::uint32_t codepoint, const unsigned char* bytes, std::size_t len) @@ -9998,15 +9976,7 @@ class view_serializer } if (codepoint <= 0x1F || (EnsureAscii && codepoint >= 0x7F)) { - if (codepoint <= 0xFFFF) - { - write_u_escape(codepoint); - } - else - { - write_u_escape(0xD7C0u + (codepoint >> 10u)); - write_u_escape(0xDC00u + (codepoint & 0x3FFu)); - } + write_u_escape(codepoint); // (a single byte: the only caller passes s[i]) return; } m_out.put(reinterpret_cast(bytes), len); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) LCOV_EXCL_LINE (printable characters are copied in runs) @@ -11828,9 +11798,11 @@ class basic_json_document { // (as before: the library's adapter decides what a null stream yields) detail::view::append_adapter(buf, detail::input_adapter(file)); - return; } - detail::view::append_file(buf, file); + else + { + detail::view::append_file(buf, file); + } } #endif diff --git a/tests/src/unit-binary_writer_sinks.cpp b/tests/src/unit-binary_writer_sinks.cpp index 1e18c3f15..7ec2eb8fc 100644 --- a/tests/src/unit-binary_writer_sinks.cpp +++ b/tests/src/unit-binary_writer_sinks.cpp @@ -14,7 +14,9 @@ using nlohmann::json; #include #include #include +#include #include +#include #include #include @@ -767,3 +769,69 @@ TEST_CASE("a write that throws leaves the same bytes in every kind of output") } } #endif + +#if !defined(JSON_NOEXCEPTION) +namespace +{ + +// an adapter whose every write fails +class failing_adapter : public nlohmann::detail::output_adapter_protocol +{ + public: + [[noreturn]] void write_character(char /*c*/) override + { + throw std::runtime_error("write failed"); + } + + [[noreturn]] void write_characters(const char* /*s*/, std::size_t /*length*/) override + { + throw std::runtime_error("write failed"); + } +}; + +} // namespace +#endif + +TEST_CASE("output adapters") +{ + SECTION("writing a single character") + { + // the writers pass whole buffers on, but an adapter takes single + // characters as well + std::vector as_vector; + nlohmann::detail::output_vector_adapter vector_adapter(as_vector); + vector_adapter.write_character('a'); + vector_adapter.write_character('b'); + CHECK(as_vector == std::vector({'a', 'b'})); + + std::ostringstream as_stream; + nlohmann::detail::output_stream_adapter stream_adapter(as_stream); + stream_adapter.write_character('c'); + stream_adapter.write_character('d'); + CHECK(as_stream.str() == "cd"); + + std::string as_string; + nlohmann::detail::output_string_adapter string_adapter(as_string); + string_adapter.write_character('e'); + string_adapter.write_character('f'); + CHECK(as_string == "ef"); + } + +#if !defined(JSON_NOEXCEPTION) + SECTION("a sink that is destroyed with characters left over") + { + // the destructor passes the pending characters on, as a write that + // threw leaves them there; the adapter failing then must not escape + { + nlohmann::detail::output_adapter_sink sink(std::make_shared()); + sink.write_character('x'); + } + + // while flush() reports the failure + nlohmann::detail::output_adapter_sink sink(std::make_shared()); + sink.write_character('x'); + CHECK_THROWS_AS(sink.flush(), std::runtime_error&); + // (the character counts as written, so there is nothing left for the destructor) + } +#endif +} diff --git a/tests/src/unit-bjdata.cpp b/tests/src/unit-bjdata.cpp index 04ee3acc4..ec7faa6cd 100644 --- a/tests/src/unit-bjdata.cpp +++ b/tests/src/unit-bjdata.cpp @@ -2312,6 +2312,23 @@ TEST_CASE("BJData") CHECK_FALSE(json::sax_parse(v, &scp, json::input_format_t::bjdata)); } + SECTION("every event of an ndarray of chars") + { + // the elements of an ndarray of 'C' are read one by one + std::vector const v = {'[', '$', 'C', '#', '[', '$', 'U', '#', 'U', 2, 2, 2, 'a', 'b', 'c', 'd'}; + // start_object, key, string, key, start_array, 2 numbers, end_array, + // key, start_array, 4 chars, end_array, end_object + constexpr int events = 16; + for (int allowed = 0; allowed < events; ++allowed) + { + CAPTURE(allowed) + SaxCountdown scp(allowed); + CHECK_FALSE(json::sax_parse(v, &scp, json::input_format_t::bjdata)); + } + SaxCountdown scp(events); + CHECK(json::sax_parse(v, &scp, json::input_format_t::bjdata)); + } + SECTION("start_array() in ndarray _ArrayData_") { std::vector const v = {'[', '$', 'U', '#', '[', 'i', 2, 'i', 3, ']', 6, 5, 4, 3, 2, 1}; @@ -3095,6 +3112,15 @@ TEST_CASE("BJData") json _; CHECK_THROWS_WITH(_ = json::from_bjdata(v), "[json.exception.parse_error.113] parse error at byte 2: syntax error while parsing BJData char: byte after 'C' must be in range 0x00..0x7F; last byte: 0x82"); } + + SECTION("ndarray of chars cut off by the end of the input") + { + // 2x2 chars announced, three present + std::vector const v = {'[', '$', 'C', '#', '[', '$', 'U', '#', 'U', 2, 2, 2, 'a', 'b', 'c'}; + json _; + CHECK_THROWS_WITH_AS(_ = json::from_bjdata(v), "[json.exception.parse_error.110] parse error at byte 16: syntax error while parsing BJData char: unexpected end of input", json::parse_error&); + CHECK(json::from_bjdata(v, true, false).is_discarded()); + } } SECTION("byte") diff --git a/tests/src/unit-bon8.cpp b/tests/src/unit-bon8.cpp index 1e24d32d4..85c9fa821 100644 --- a/tests/src/unit-bon8.cpp +++ b/tests/src/unit-bon8.cpp @@ -329,6 +329,9 @@ TEST_CASE("BON8") check_bon8({false, nullptr, true}, {0x83, 0xF8, 0xFA, 0xF9}); check_bon8({false, nullptr, true, 1.0}, {0x84, 0xF8, 0xFA, 0xF9, 0xFD}); check_bon8({false, nullptr, true, 1.0, json::array(), 0.0}, {0x85, 0xF8, 0xFA, 0xF9, 0xFD, 0x80, 0xFC, 0xFE}); + // an array that starts with a number: -1, 0, and 1 have bytes of their own + check_bon8({-1.0, 0.0, 1.0}, {0x83, 0xFB, 0xFC, 0xFD}); + check_bon8({1.0, -1.0, -0.0}, {0x83, 0xFD, 0xFB, 0x8E, 0x80, 0x00, 0x00, 0x00}); check_bon8({{{1}}}, {0x81, 0x81, 0x81, 0x91}); check_bon8({{{"foo"}}}, {0x81, 0x81, 0x81, 'f', 'o', 'o', 0xFF}); check_bon8({{{""}}}, {0x81, 0x81, 0x81, 0xFF}); diff --git a/tests/src/unit-bson.cpp b/tests/src/unit-bson.cpp index ac8e2b24f..f7957d566 100644 --- a/tests/src/unit-bson.cpp +++ b/tests/src/unit-bson.cpp @@ -82,8 +82,20 @@ class huge_string_t : public std::string return result; } + // returns a copy of @a s whose size() pretends to be @a n + static huge_string_t as_size(const std::string& s, const size_type n) + { + huge_string_t result(s); + result.fake_size = n; + return result; + } + size_type size() const noexcept { + if (fake_size != 0) + { + return fake_size; + } if (pretend_huge) { // one byte more than the BSON length field can represent @@ -94,6 +106,7 @@ class huge_string_t : public std::string private: bool pretend_huge = false; + size_type fake_size = 0; }; using huge_string_json = nlohmann::json::with_string_t; @@ -268,6 +281,17 @@ TEST_CASE("BSON") CHECK_THROWS_WITH_AS(huge_string_json::to_bson(j), "[json.exception.out_of_range.412] BSON length 2147483661 exceeds maximum of 2147483647", huge_string_json::out_of_range&); } + SECTION("an array element that no longer fits the document") + { + // the string fits the length field, but not the document that + // holds it; nothing of it is written + const auto limit = static_cast((std::numeric_limits::max)()); + huge_string_json j; + j["a"] = huge_string_json::array({huge_string_t::as_size("value", limit - 8)}); + + CHECK_THROWS_AS(huge_string_json::to_bson(j), huge_string_json::out_of_range&); + } + SECTION("document") { // an oversized string nested one level deep makes the @@ -899,6 +923,38 @@ TEST_CASE("BSON") CHECK_THROWS_AS(_ = json::from_bson(input), json::parse_error&); } + SECTION("binary member with subtype 0x02 (old binary) with a truncated inner length") + { + // the inner length field is cut off by the end of the input + std::vector const input = + { + 0x0F, 0x00, 0x00, 0x00, // size (little endian) + 0x05, 'a', 0x00, // entry: binary + 0x08, 0x00, 0x00, 0x00, // size of binary (little endian) + 0x02, // "old binary" subtype + 0x01 // first byte of the inner length + }; + + json _; + CHECK_THROWS_WITH_AS(_ = json::from_bson(input), "[json.exception.parse_error.110] parse error at byte 14: syntax error while parsing BSON number: unexpected end of input", json::parse_error&); + CHECK(json::from_bson(input, true, false).is_discarded()); + } + + SECTION("string member that is cut off by the end of the input") + { + std::vector const input = + { + 0x0F, 0x00, 0x00, 0x00, // size (little endian) + 0x02, 'a', 0x00, // entry: string + 0x0A, 0x00, 0x00, 0x00, // size of the string (little endian) + 'b', 'c' // the string ends early + }; + + json _; + CHECK_THROWS_WITH_AS(_ = json::from_bson(input), "[json.exception.parse_error.110] parse error at byte 14: syntax error while parsing BSON string: unexpected end of input", json::parse_error&); + CHECK(json::from_bson(input, true, false).is_discarded()); + } + SECTION("Some more complex document") { json const j = @@ -2174,6 +2230,25 @@ value = depth % 2 == 0 ? json{{"a", std::move(value)}, {"b", {1, "x"}}} : CHECK(output.empty()); } +#if !defined(JSON_NOEXCEPTION) + SECTION("U+0000 in a long key is found at any position") + { + // the key is searched eight characters at a time + for (const std::string& key : + { + std::string("abc\0defghijkl", 12), std::string("abcdefgh\0ijkl", 13), std::string("abcdefghijklmnop\0", 17), std::string("\0abcdefghijkl", 13), + std::string("abcdefghijklmno\0", 16) + }) + { + CAPTURE(key) + const json value = json::object({{key, 1}}); + std::vector output; + CHECK_THROWS_AS(json::to_bson(value, output), json::out_of_range&); + CHECK(output.empty()); + } + } +#endif + SECTION("a binary subtype that doesn't fit a byte is rejected before anything is written (#5675)") { // the offending value is nested, so this also covers that the check diff --git a/tests/src/unit-cbor.cpp b/tests/src/unit-cbor.cpp index 930cbe7f5..442050cf4 100644 --- a/tests/src/unit-cbor.cpp +++ b/tests/src/unit-cbor.cpp @@ -1809,6 +1809,30 @@ TEST_CASE("CBOR") CHECK(json::from_cbor(std::vector({0xa1, 0xff, 0x01}), true, false).is_discarded()); } + SECTION("tag in front of a key that is cut off by the end of the input") + { + // 0xD8: the tag number is in the next byte, which is missing + const std::vector input = {0xA1, 0xD8}; + json _; + for (const auto handler : + { + json::cbor_tag_handler_t::ignore, json::cbor_tag_handler_t::store + }) + { + CHECK_THROWS_WITH_AS(_ = json::from_cbor(input, true, true, handler), "[json.exception.parse_error.110] parse error at byte 3: syntax error while parsing CBOR number: unexpected end of input", json::parse_error&); + CHECK(json::from_cbor(input, true, false, handler).is_discarded()); + } + } + + SECTION("chunk of an indefinite-length string that is cut off by the end of the input") + { + // the second byte announces a chunk of three bytes, but two follow + const std::vector input = {0x7F, 0x63, 'a', 'b'}; + json _; + CHECK_THROWS_WITH_AS(_ = json::from_cbor(input), "[json.exception.parse_error.110] parse error at byte 5: syntax error while parsing CBOR string: unexpected end of input", json::parse_error&); + CHECK(json::from_cbor(input, true, false).is_discarded()); + } + SECTION("tag in front of a key") { const auto ignore = json::cbor_tag_handler_t::ignore; diff --git a/tests/src/unit-class_lexer.cpp b/tests/src/unit-class_lexer.cpp index 2926ef88b..0e8b22317 100644 --- a/tests/src/unit-class_lexer.cpp +++ b/tests/src/unit-class_lexer.cpp @@ -2092,6 +2092,12 @@ TEST_CASE("UTF-8 validators accept the same sequences") } }; + // an ASCII byte is a sequence of its own + for (unsigned lead = 0; lead < 0x80; ++lead) + { + CHECK(nlohmann::detail::utf8_shape(static_cast(lead)).length == 1); + } + for (unsigned lead = 0x80; lead <= 0xFF; ++lead) { const auto shape = nlohmann::detail::utf8_shape(static_cast(lead)); @@ -2157,3 +2163,72 @@ TEST_CASE("UTF-8 validators accept the same sequences") } } } + +TEST_CASE("float_bigint") +{ + // the big integer of the exact comparison of a float token with the + // midpoint between two floats (digit_comparison()) + using nlohmann::detail::float_bigint; + + SECTION("zero") + { + // shifting zero keeps it zero, whatever the distance + float_bigint zero(0); + zero.shift_left(0); + zero.shift_left(5); + zero.shift_left(64); + CHECK(zero.compare(float_bigint(0)) == 0); + CHECK(zero.compare(float_bigint(1)) == -1); + CHECK(float_bigint(1).compare(zero) == 1); + } + + SECTION("compare numbers of different lengths") + { + // the number with more 32-bit limbs is the greater one + const float_bigint small(0xFFFFFFFFu); + const float_bigint large(std::uint64_t{1} << 32u); + CHECK(small.compare(large) == -1); + CHECK(large.compare(small) == 1); + CHECK(large.compare(large) == 0); + + // a shift can add a limb + float_bigint shifted(1); + shifted.shift_left(32); + CHECK(shifted.compare(large) == 0); + shifted.shift_left(1); + CHECK(shifted.compare(large) == 1); + CHECK(large.compare(shifted) == -1); + } + + SECTION("compare numbers of the same length") + { + CHECK(float_bigint(2).compare(float_bigint(3)) == -1); + CHECK(float_bigint(3).compare(float_bigint(2)) == 1); + CHECK(float_bigint(3).compare(float_bigint(3)) == 0); + CHECK(float_bigint((std::uint64_t{3} << 32u) | 1u).compare(float_bigint((std::uint64_t{3} << 32u) | 2u)) == -1); + } +} + +TEST_CASE("serialization of short strings") +{ + // the serializer takes strings of up to 32 characters that need no + // escaping by a table lookup of its own where the scan is not vectorized + // (this file is built that way, too, with JSON_NO_SIMD) + for (const std::string& s : + { + std::string(), std::string("a"), std::string("hello"), std::string("0123456789abcdef"), std::string("0123456789abcdef0123456789abcdef"), + std::string("0123456789abcdef0123456789abcdef0") + }) + { + CAPTURE(s) + const json j = s; + CHECK(j.dump() == "\"" + s + "\""); + CHECK(j.dump(-1, ' ', true) == "\"" + s + "\""); + CHECK(json({{s, s}}).dump() == "{\"" + s + "\":\"" + s + "\"}"); + } + + // a character that needs an escape + CHECK(json("a\"b").dump() == "\"a\\\"b\""); + CHECK(json("caf\xC3\xA9").dump() == "\"caf\xC3\xA9\""); + CHECK(json("caf\xC3\xA9").dump(-1, ' ', true) == "\"caf\\u00e9\""); +} diff --git a/tests/src/unit-custom-array-type.cpp b/tests/src/unit-custom-array-type.cpp index e602b4716..65285eb54 100644 --- a/tests/src/unit-custom-array-type.cpp +++ b/tests/src/unit-custom-array-type.cpp @@ -210,3 +210,23 @@ TEST_CASE("regression test #5476 - array type without reserve()") CHECK(json::from_cbor(truncated, true, false).is_discarded()); } } + +TEST_CASE("array type with std::deque nested beyond the recursion limit") +{ + // the binary writers finish a value nested deeper than + // recursion_depth_limit() with an explicit stack of frames; with the + // iterators of std::deque, these frames are copied and moved through + // their own special members + deque_json j = 1; + for (int i = 0; i < 200; ++i) + { + deque_json outer = deque_json::array(); + outer.push_back(std::move(j)); + j = std::move(outer); + } + + // 200 arrays of one element each (0x81) around the integer 1 + std::vector expected(200, 0x81); + expected.push_back(0x01); + CHECK(deque_json::to_cbor(j) == expected); +} diff --git a/tests/src/unit-custom-key-type-full.cpp b/tests/src/unit-custom-key-type-full.cpp index b72bd7ae0..69a052c9f 100644 --- a/tests/src/unit-custom-key-type-full.cpp +++ b/tests/src/unit-custom-key-type-full.cpp @@ -49,3 +49,24 @@ TEST_CASE("custom object key types: MessagePack (json_full)") { custom_key_test::test_msgpack(); } + +TEST_CASE("custom object key types: patch (json_full)") +{ + // the members of a patch are found by lookup, as their keys are not std::string + const json_full j = json_full::parse(R"({"a":1,"b":[1,2],"c":{"d":2}})"); + const json_full patch = json_full::parse(R"([ + {"op": "add", "path": "/e", "value": 5}, + {"op": "remove", "path": "/a"}, + {"op": "replace", "path": "/c/d", "value": 3}, + {"op": "test", "path": "/c/d", "value": 3}, + {"op": "move", "from": "/e", "path": "/f"}, + {"op": "copy", "from": "/f", "path": "/g"}, + {"op": "add", "path": "/b/-", "value": 7} + ])"); + + CHECK(j.patch(patch).dump() == R"({"b":[1,2,7],"c":{"d":3},"f":5,"g":5})"); + + // a patch with a missing member is rejected + CHECK_THROWS_AS(j.patch(json_full::parse(R"([{"op": "add", "path": "/e"}])")), json_full::parse_error&); + CHECK_THROWS_AS(j.patch(json_full::parse(R"([{"op": "test", "path": "/a", "value": 2}])")), json_full::other_error&); +} diff --git a/tests/src/unit-custom-number-types.cpp b/tests/src/unit-custom-number-types.cpp index e324417db..58353d67e 100644 --- a/tests/src/unit-custom-number-types.cpp +++ b/tests/src/unit-custom-number-types.cpp @@ -337,6 +337,52 @@ TEST_CASE("custom number types: integers beyond 64 bits") CHECK(wide_json::to_bson(fit_members) == nlohmann::json::to_bson(fit_members64)); } + SECTION("MessagePack integers of every width") + { + // an integer of a number type wider than 64 bits is written with the + // smallest MessagePack integer that holds it, as for 64-bit types + const wide_json widths = wide_json::array( + { + make_int(-33), make_int(-100), make_int(-128), make_int(-129), make_int(-32768), make_int(-32769), make_int(-2147483648LL), make_int(-2147483649LL), + make_int(128), make_int(255), make_int(256), make_int(300), make_int(65535), make_int(65536), make_int(70000), make_int(4294967295LL), make_int(4294967296LL), + make_uint(128), make_uint(255), make_uint(256), make_uint(300), make_uint(65535), make_uint(65536), make_uint(70000), make_uint(4294967295ULL), make_uint(4294967296ULL) + }); + const nlohmann::json widths64 = nlohmann::json::array( + { + -33, -100, -128, -129, -32768, -32769, -2147483648LL, -2147483649LL, + 128, 255, 256, 300, 65535, 65536, 70000, 4294967295LL, 4294967296LL, + 128u, 255u, 256u, 300u, 65535u, 65536u, 70000u, 4294967295ULL, 4294967296ULL + }); + CHECK(wide_json::to_msgpack(widths) == nlohmann::json::to_msgpack(widths64)); + + // as single values, too + CHECK(wide_json::to_msgpack(make_int(-100)) == std::vector({0xD0, 0x9C})); + CHECK(wide_json::to_msgpack(make_int(300)) == std::vector({0xCD, 0x01, 0x2C})); + CHECK(wide_json::to_msgpack(make_int(70000)) == std::vector({0xCE, 0x00, 0x01, 0x11, 0x70})); + CHECK(wide_json::to_msgpack(make_uint(300)) == std::vector({0xCD, 0x01, 0x2C})); + CHECK(wide_json::to_msgpack(make_uint(70000)) == std::vector({0xCE, 0x00, 0x01, 0x11, 0x70})); + } + + SECTION("dump") + { + // beyond 64 bits, the digits are counted before they are written + CHECK(make_int(0).dump() == "0"); + CHECK(make_int(two_64).dump() == "18446744073709551616"); + CHECK(make_int(-two_64).dump() == "-18446744073709551616"); + CHECK(make_int(two_100).dump() == "1267650600228229401496703205376"); + CHECK(make_int(-two_100).dump() == "-1267650600228229401496703205376"); + // powers of ten, where the count of digits ends on a single digit or two + const int128 ten_20 = static_cast(100000000000ULL) * static_cast(1000000000ULL); + CHECK(make_int(ten_20).dump() == "100000000000000000000"); + CHECK(make_int(-ten_20).dump() == "-100000000000000000000"); + CHECK(make_int(ten_20 * 10).dump() == "1000000000000000000000"); + CHECK(make_int(ten_20 * 100).dump() == "10000000000000000000000"); + CHECK(make_int(ten_20 * 1000).dump() == "100000000000000000000000"); + CHECK(make_uint(static_cast(ten_20) * 10u).dump() == "1000000000000000000000"); + CHECK(make_uint((std::numeric_limits::max)()).dump() == "340282366920938463463374607431768211455"); + CHECK(make_int((std::numeric_limits::min)()).dump() == "-170141183460469231731687303715884105728"); + } + SECTION("BJData ND-array") { // an ND-array element beyond 64 bits does not fit any dtype, so the @@ -417,3 +463,23 @@ TEST_CASE("custom number types") } } + +TEST_CASE("custom number types: 32-bit integers in the binary formats") +{ + // integers narrower than 64 bits are widened (a signed one with its sign) + // before the CBOR, MessagePack, and UBJSON writers pick their encoding + using json32 = nlohmann::json::with_integers_t; + json32 values = json32::array(); + values.push_back(std::uint32_t{0}); + values.push_back(std::uint32_t{200}); + values.push_back(std::uint32_t{4294967295u}); + values.push_back(std::int32_t{-1}); + values.push_back(std::int32_t{-2147483647 - 1}); + REQUIRE(values[2].is_number_unsigned()); + REQUIRE(values[4].is_number_integer()); + + const nlohmann::json values64 = {0u, 200u, 4294967295u, -1, -2147483648LL}; + CHECK(json32::to_cbor(values) == nlohmann::json::to_cbor(values64)); + CHECK(json32::to_msgpack(values) == nlohmann::json::to_msgpack(values64)); + CHECK(json32::to_ubjson(values) == nlohmann::json::to_ubjson(values64)); +} diff --git a/tests/src/unit-custom-object-key-type.cpp b/tests/src/unit-custom-object-key-type.cpp index 9c64ebce0..acb78d628 100644 --- a/tests/src/unit-custom-object-key-type.cpp +++ b/tests/src/unit-custom-object-key-type.cpp @@ -11,8 +11,13 @@ #include #include +#include +#include +#include #include +#include #include +#include #include "custom_object_key_type.hpp" @@ -125,3 +130,161 @@ TEST_CASE("MessagePack supports custom object key types nested deeper than the r CHECK(encoded == nlohmann::json::to_msgpack(expected)); CHECK(nlohmann::json::from_msgpack(encoded) == expected); } + +namespace +{ +// A key type that cannot be constructed from a std::string, so that a value +// with such keys cannot be converted from a nlohmann::json by the range +// constructor of its object type: the serializer converts the object instead. +class strict_key +{ + public: + strict_key(const char* value) + : m_value(value) + {} + + strict_key(const std::string& /*value*/) = delete; + +#if JSON_DIAGNOSTICS + // Required by JSON_DIAGNOSTICS, which reads object keys through data() + // when building the path of an exception. + const char* data() const noexcept + { + return m_value.data(); + } +#endif + + friend bool operator<(const strict_key& lhs, const strict_key& rhs) + { + return lhs.m_value < rhs.m_value; + } + + private: + std::string m_value{}; +}; + +strict_key make_strict_key(const std::string& key) +{ + return strict_key(key.c_str()); +} + +template +class strict_object + : public std::map < + strict_key, + Value, + std::less, // NOLINT(modernize-use-transparent-functors) + typename std::allocator_traits::template rebind_alloc < + std::pair>> +{ + public: + strict_object() = default; +}; + +template +struct strict_serializer : nlohmann::adl_serializer {}; + +using strict_json = nlohmann::basic_json < strict_object, std::vector, std::string, bool, std::int64_t, + std::uint64_t, double, std::allocator, strict_serializer, std::vector>; + +// the members of an object are converted one by one, their keys by make_strict_key() +template<> +struct strict_serializer +{ + static void to_json(strict_json& j, const nlohmann::json::object_t& object) + { + strict_json::object_t converted; + for (const auto& member : object) + { + converted.emplace(make_strict_key(member.first), strict_json(member.second)); + } + j = strict_json(std::move(converted)); + } +}; +} // namespace + +TEST_CASE("converting a value with keys that cannot be made from a std::string nested deeper than the recursion depth limit") +{ + static_assert(!std::is_constructible::value, "the keys cannot be made from a std::string"); + + // below detail::recursion_depth_limit(), objects and arrays are converted + // by the serializers instead of the range constructors + const std::size_t depth = nlohmann::detail::recursion_depth_limit() + 10; + + nlohmann::json value = 1; + for (std::size_t i = 0; i < depth; ++i) + { + if (i % 2 == 0) + { + value = nlohmann::json::object({{"a", std::move(value)}, {"b", true}}); + } + else + { + value = nlohmann::json::array({std::move(value), "sibling"}); + } + } + + const strict_json converted = value; + + // at() needs to make a std::string of a key for its error message + const auto member = [](const strict_json & object, const char* name) -> const strict_json & + { + const auto it = object.find(strict_key(name)); + REQUIRE(it != object.end()); + return *it; + }; + + // walk down the nesting levels without recursion + const strict_json* current = &converted; + for (std::size_t i = depth; i > 0; --i) + { + if (i % 2 == 1) + { + REQUIRE(current->is_object()); + CHECK(current->size() == 2); + CHECK(member(*current, "b").get()); + current = &member(*current, "a"); + } + else + { + REQUIRE(current->is_array()); + CHECK(current->size() == 2); + CHECK(current->at(1).get() == "sibling"); + current = ¤t->at(0); + } + } + CHECK(current->get() == 1); +} + +namespace +{ +// shorter keys come first +struct shortlex_less +{ + bool operator()(const std::string& lhs, const std::string& rhs) const + { + return lhs.size() != rhs.size() ? lhs.size() < rhs.size() : lhs < rhs; + } +}; + +template +using shortlex_map = std::map; + +using shortlex_json = nlohmann::json::with_object_t; +} // namespace + +TEST_CASE("unflatten with an object type that does not sort the keys alphabetically") +{ + // the keys of /a come before and after one of /b: unlike in an object + // sorted alphabetically, the values below a prefix are not found one + // after the other, so the value of /a must be looked up again + shortlex_json flat = shortlex_json::object(); + flat["/a/p"] = 1; + flat["/b/qq"] = 2; + flat["/a/pqr"] = 3; + REQUIRE(flat.begin().key() == "/a/p"); + REQUIRE((++flat.begin()).key() == "/b/qq"); + + CHECK(flat.unflatten().dump() == R"({"a":{"p":1,"pqr":3},"b":{"qq":2}})"); + CHECK(flat.unflatten().flatten() == flat); +} diff --git a/tests/src/unit-deserialization.cpp b/tests/src/unit-deserialization.cpp index e82685311..5b8f086fd 100644 --- a/tests/src/unit-deserialization.cpp +++ b/tests/src/unit-deserialization.cpp @@ -1195,6 +1195,28 @@ TEST_CASE("deserialization") #endif } +#ifndef JSON_TEST_DEPRECATED_FUNCTIONS_DELETED +TEST_CASE("deserialization from a pointer and a length (deprecated)") +{ + // the unit test build disables -Wdeprecated-declarations (see + // cmake/clang_flags.cmake) + const std::string text = R"({"foo": [1, 2, 3]})"; + const std::string invalid = R"({"foo": )"; + + SECTION("parse") + { + const json j = json::parse({text.data(), text.size()}); + CHECK(j == json({{"foo", {1, 2, 3}}})); + } + + SECTION("accept") + { + CHECK(json::accept({text.data(), text.size()})); + CHECK_FALSE(json::accept({invalid.data(), invalid.size()})); + } +} +#endif + // select the types to test - char8_t is only available since C++20 if and only // if __cpp_char8_t is defined. #define TYPE_LIST(...) __VA_ARGS__ diff --git a/tests/src/unit-diagnostics.cpp b/tests/src/unit-diagnostics.cpp index 504a95322..2987d348c 100644 --- a/tests/src/unit-diagnostics.cpp +++ b/tests/src/unit-diagnostics.cpp @@ -868,3 +868,23 @@ TEST_CASE("Parent pointers stay consistent for object types not known to keep th obj["1"].push_back(42); CHECK_THROWS_WITH_AS(obj["1"].at(5), "[json.exception.out_of_range.401] (/1) array index 5 is out of range", relocating_json::out_of_range&); } + +TEST_CASE("patch_inplace() with a patch that is stored inside the patched value") +{ + // the patch is modified-while-iterated if it is not copied first: it is a + // descendant of the value it is applied to, which only the parent pointers + // (JSON_DIAGNOSTICS) reveal + json j = + { + {"patch", {{{"op", "add"}, {"path", "/x"}, {"value", 1}}, {{"op", "remove"}, {"path", "/patch/0"}}}}, + {"y", 2} + }; + + j.patch_inplace(j["patch"]); + + CHECK(j.size() == 3); + CHECK(j["x"] == 1); + CHECK(j["y"] == 2); + CHECK(j["patch"].size() == 1); + CHECK(j["patch"][0]["op"] == "remove"); +} diff --git a/tests/src/unit-iterators2.cpp b/tests/src/unit-iterators2.cpp index 5bc738e21..67a5752e4 100644 --- a/tests/src/unit-iterators2.cpp +++ b/tests/src/unit-iterators2.cpp @@ -18,6 +18,8 @@ #include using nlohmann::json; +#include + #if JSON_HAS_RANGES #include #include @@ -665,6 +667,23 @@ TEST_CASE("iterators 2") } } + SECTION("reverse iterators of a const value") + { + const json j_array = {1, 2, 3}; + const json j_object = {{"one", 1}, {"two", 2}}; + + // rbegin() and rend() of a const value are crbegin() and crend() + CHECK(j_array.rbegin() == j_array.crbegin()); + CHECK(j_array.rend() == j_array.crend()); + CHECK(*j_array.rbegin() == 3); + CHECK(std::distance(j_array.rbegin(), j_array.rend()) == 3); + + CHECK(j_object.rbegin() == j_object.crbegin()); + CHECK(j_object.rend() == j_object.crend()); + CHECK(j_object.rbegin().key() == "two"); + CHECK(std::distance(j_object.rbegin(), j_object.rend()) == 2); + } + SECTION("reverse iterator arithmetic") { json j_object = {{"one", 1}, {"two", 2}, {"three", 3}}; diff --git a/tests/src/unit-json_patch.cpp b/tests/src/unit-json_patch.cpp index 83df15350..37c6e29d9 100644 --- a/tests/src/unit-json_patch.cpp +++ b/tests/src/unit-json_patch.cpp @@ -14,9 +14,11 @@ using nlohmann::json; using namespace nlohmann::literals; // NOLINT(google-build-using-namespace) #endif +#include #include #include #include +#include #include #include "make_test_data_available.hpp" @@ -2453,3 +2455,99 @@ TEST_CASE("JSON patch - diff behavior documented in the API reference") CHECK(json::diff(json(nullptr), json::array()) == R"([{"op": "replace", "path": "", "value": []}])"_json); } } + +TEST_CASE("JSON patch - operations that are left to json_pointer") +{ + // the usual operations on a plain pointer are applied without building a + // json_pointer; the following are not plain, and some are errors + SECTION("a path without a slash") + { + const json j = {{"a", 1}}; + + CHECK_THROWS_AS(j.patch(R"([{"op": "add", "path": "abc", "value": 1}])"_json), json::parse_error&); + CHECK_THROWS_AS(j.patch(R"([{"op": "remove", "path": "abc"}])"_json), json::parse_error&); + } + + SECTION("remove with an escaped reference token") + { + const json j = {{"a/b", 1}, {"c~d", 2}, {"e", 3}}; + + CHECK(j.patch(R"([{"op": "remove", "path": "/a~1b"}])"_json) == json({{"c~d", 2}, {"e", 3}})); + CHECK(j.patch(R"([{"op": "remove", "path": "/c~0d"}])"_json) == json({{"a/b", 1}, {"e", 3}})); + CHECK_THROWS_AS(j.patch(R"([{"op": "remove", "path": "/missing~1key"}])"_json), json::out_of_range&); + } + + SECTION("a path through a primitive value") + { + const json j = {{"a", 1}}; + + CHECK_THROWS_AS(j.patch(R"([{"op": "test", "path": "/a/b", "value": 1}])"_json), json::other_error&); + CHECK_THROWS_AS(j.patch(R"([{"op": "replace", "path": "/a/b", "value": 1}])"_json), json::out_of_range&); + } +} + +TEST_CASE("JSON patch - diff of objects with many members") +{ + // more members than diff() keeps without allocating + for (const std::size_t members : + { + 16u, 17u, 40u + }) + { + CAPTURE(members) + + json source = json::object(); + json target = json::object(); + for (std::size_t i = 0; i < members; ++i) + { + const std::string key = "key" + std::to_string(i); + source[key] = i; + // every third member differs + target[key] = (i % 3 == 0) ? i + 1 : i; + } + // the same keys, a removed and an added one + source["removed"] = true; + target["added"] = false; + + const json patch = json::diff(source, target); + CHECK(source.patch(patch) == target); + CHECK(patch.size() == ((members + 2) / 3) + 2); + + const nlohmann::ordered_json ordered_source = source; + const nlohmann::ordered_json ordered_target = target; + CHECK(ordered_source.patch(nlohmann::ordered_json::diff(ordered_source, ordered_target)) == ordered_target); + } +} + +TEST_CASE("JSON patch - diff of leaf values below the bound of the recursive descent") +{ + // the values at the innermost level are diffed without recursion: equal + // leaves yield nothing, leaves of different types and different leaves + // are replaced + const std::size_t depth = 200; + + json source = {1, "text", 3, {{"x", 1}, {"y", 2}}, true, nullptr}; + json target = {1, 2, 4, {{"x", 1}, {"y", 3}}, true, nullptr}; + for (std::size_t i = 0; i < depth; ++i) + { + source = json::array({std::move(source)}); + target = json::array({std::move(target)}); + } + + const json patch = json::diff(source, target); + CHECK(patch.size() == 3); + CHECK(source.patch(patch) == target); + + // the same with objects as the outer levels + json object_source = {{"equal", 1}, {"type", "text"}, {"value", 3}}; + json object_target = {{"equal", 1}, {"type", 2}, {"value", 4}}; + for (std::size_t i = 0; i < depth; ++i) + { + object_source = json::object({{"a", std::move(object_source)}}); + object_target = json::object({{"a", std::move(object_target)}}); + } + + const json object_patch = json::diff(object_source, object_target); + CHECK(object_patch.size() == 2); + CHECK(object_source.patch(object_patch) == object_target); +} diff --git a/tests/src/unit-json_pointer.cpp b/tests/src/unit-json_pointer.cpp index 0e225b86e..8afb52f48 100644 --- a/tests/src/unit-json_pointer.cpp +++ b/tests/src/unit-json_pointer.cpp @@ -106,6 +106,12 @@ TEST_CASE("JSON pointers") CHECK_THROWS_WITH_AS(p.pop_back(), "[json.exception.out_of_range.405] JSON pointer has no parent", json::out_of_range&); + // the top of a pointer is its first reference token + const json::json_pointer nonempty("/foo/bar/0"); + CHECK(nonempty.top() == json::json_pointer("/foo")); + CHECK(nonempty.top().to_string() == "/foo"); + CHECK(json::json_pointer("/a~1b").top() == json::json_pointer("/a~1b")); + SECTION("array index error") { json v = {1, 2, 3, 4}; @@ -697,6 +703,16 @@ TEST_CASE("JSON pointers") CHECK_THROWS_WITH_AS(j_error.unflatten(), "[json.exception.type_error.313] invalid value to unflatten", json::type_error&); + // error for an invalid escape sequence in a key + CHECK_THROWS_WITH_AS(json({{"/a~2", 1}}).unflatten(), + "[json.exception.parse_error.108] parse error: escape character '~' must be followed with '0' or '1'", json::parse_error&); + CHECK_THROWS_WITH_AS(json({{"/a/b", 1}, {"/c~", 1}}).unflatten(), + "[json.exception.parse_error.108] parse error: escape character '~' must be followed with '0' or '1'", json::parse_error&); + + // error for an element of an array that is also a container + CHECK_THROWS_WITH_AS(json({{"/a/0", 1}, {"/a/0/x", 2}}).unflatten(), + "[json.exception.type_error.313] invalid value to unflatten", json::type_error&); + // explicit roundtrip check CHECK(j.flatten().unflatten() == j); diff --git a/tests/src/unit-json_view.cpp b/tests/src/unit-json_view.cpp index 8dc60e912..e383dbdac 100644 --- a/tests/src/unit-json_view.cpp +++ b/tests/src/unit-json_view.cpp @@ -31,9 +31,11 @@ using json_view_test::has_duplicate_keys; #include #include #include +#include #include #include #include +#include #include #include #include @@ -98,6 +100,37 @@ template using bool_conversion_t = decltype(static_cast(std::declval())); #if !defined(JSON_NOEXCEPTION) +// an allocator that never has room +template +struct failing_allocator +{ + using value_type = T; + + failing_allocator() = default; + + template + explicit failing_allocator(const failing_allocator& /*other*/) noexcept {} + + T* allocate(std::size_t /*n*/) // NOLINT(readability-convert-member-functions-to-static) + { + throw std::bad_alloc(); + } + + void deallocate(T* /*p*/, std::size_t /*n*/) noexcept {} // NOLINT(readability-convert-member-functions-to-static) + + template + bool operator==(const failing_allocator& /*other*/) const noexcept + { + return true; + } + + template + bool operator!=(const failing_allocator& /*other*/) const noexcept + { + return false; + } +}; + // the exception parse() throws for a text, or "" if it accepts it std::string parse_exception(const std::string& text, bool comments = false, bool trailing_commas = false) { @@ -235,7 +268,9 @@ TEST_CASE("json_view") { "", " ", "[", "]", "{", "[1,]", "{\"a\":1,}", "[1 2]", "{\"a\" 1}", "{1:2}", "tru", "nul", "fals", "truex", "-", "01", "1.", ".5", "1e", "\"", "\"abc", "\"\\x\"", "\"\\u12\"", "\"\\ud800\"", "\"\\udc00\"", "\"\x01\"", "\"\xff\"", "\"\xc3\"", "[1]x", "/", "/*", "[\n 1,\n x\n]", // NOLINT(modernize-raw-string-literal) - "1e400", "-1e400", "[1.7976931348623159e308]", "{\"a\":\n{\"b\": [1, 2,\n 3 x]}}" + "1e400", "-1e400", "[1.7976931348623159e308]", "{\"a\":\n{\"b\": [1, 2,\n 3 x]}}", + // (an error after every kind of value: the error is found by replaying the parser) + "[null, true, -1, 1, 1.5, \"s\", {\"a\": 1} 2]", "{\"a\": null, \"b\": -1, \"c\": 1.5, \"d\": \"s\", \"e\": {}, 1}" }) { CAPTURE(text) @@ -368,6 +403,10 @@ TEST_CASE("json_view") CHECK(from_stream.root().materialize() == expected); const std::list list(text.begin(), text.end()); CHECK(json_document::parse(list.begin(), list.end()).owns_source()); + // (an input that is not valid) + const std::list bad_list{'[', '1', ','}; + json_document from_bad_list; + CHECK_THROWS_AS(from_bad_list = json_document::parse(bad_list.begin(), bad_list.end()), json::parse_error&); CHECK(materialized(list.begin(), list.end()) == expected); // iterator pairs: pointers are borrowed, and so are contiguous library @@ -1511,6 +1550,17 @@ TEST_CASE("json_view values") const json_document dup = json_document::parse(R"({"a":1,"a":2})"); CHECK((dup.root().get>() == std::map {{"a", 2}})); + // a map of views from something that is not an object + using view_map = std::map; + const json_document not_an_object = json_document::parse("[1,2]"); + CHECK_THROWS_WITH_AS(not_an_object.root().get(), json_view_test::message("type_error.302", "", 0, 5, "type must be object, but is array").c_str(), json::type_error&); +#if !defined(JSON_NOEXCEPTION) + // (and one that cannot allocate its nodes) + using failing_map = std::map, failing_allocator>>; + const json_document one_member = json_document::parse(R"({"a":1})"); + CHECK_THROWS_AS(one_member.root().get(), std::bad_alloc&); +#endif + const json_view invalid{}; CHECK_THROWS_WITH_AS(invalid.get(), "[json.exception.type_error.302] type must be number, but is discarded", json::type_error&); CHECK(invalid.get().is_discarded()); diff --git a/tests/src/unit-json_view_compare_values.cpp b/tests/src/unit-json_view_compare_values.cpp new file mode 100644 index 000000000..132978d8d --- /dev/null +++ b/tests/src/unit-json_view_compare_values.cpp @@ -0,0 +1,83 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#include "doctest_compatibility.h" + +#include +using nlohmann::json; +using nlohmann::json_document; + +#include +#include +#include + +// These tests were split off unit-json_view_compare.cpp, whose object file got +// too large for the MinGW assembler ("file too big"). + +namespace +{ +// an object type that does not enumerate its members in ascending order of keys +template +using descending_object = std::map, Allocator>; +} // namespace + +TEST_CASE("json_view comparison of differing values") +{ + SECTION("values that differ in their first bytes, or are written differently") + { + const auto same = [](const char* x, const char* y) + { + const json_document dx = json_document::parse(x); + const json_document dy = json_document::parse(y); + const bool equal = dx.root() == dy.root(); + CHECK(equal == (json::parse(x) == json::parse(y))); + CHECK(equal == (dx.root() == json::parse(y))); + const nlohmann::json_editable_document ex = nlohmann::json_editable_document::parse(x); + CHECK(equal == (ex.root() == dy.root())); + return equal; + }; + // booleans + CHECK(!same("true", "false")); + CHECK(!same("[true]", "[false]")); + CHECK(!same(R"({"a":false,"b":[1]})", R"({"a":true,"b":[1]})")); + CHECK(same(R"({"a":true,"b":[1]})", R"({"a":true,"b":[1]})")); + // strings and keys of equal length that differ in the first eight bytes (they are compared in words) + CHECK(!same(R"("abcdefghijklmnopqrstuvwxyz")", R"("Xbcdefghijklmnopqrstuvwxyz")")); + CHECK(!same(R"(["abcdefghijklmnopq",1])", R"(["abcdefghXjklmnopq",1])")); + CHECK(!same(R"({"abcdefghijklmnopq":1})", R"({"Xbcdefghijklmnopq":1})")); + CHECK(!same(R"({"abcdefghijklmnopq":[1]})", R"({"abcdefghijklmnopX":[1]})")); + // floats of one layout and different tokens are the same value + CHECK(same("[1.5e0, 2]", "[1.5E0, 2]")); + CHECK(same("[1.5e0, 2]", "[1.5e+0, 2]")); + CHECK(same("[1.50, 2]", "[1.5, 2]")); + CHECK(!same("[1.5e0, 2]", "[1.6e0, 2]")); + CHECK(!same("[1.5e0, 2]", "[1.5E1, 2]")); + CHECK(same(R"({"a":1.25e0,"b":[3]})", R"({"a":1.25E0,"b":[3]})")); + } + + SECTION("a basic_json whose members are not sorted") + { + // (an object type that is not an ordered map, and does not keep its members sorted either) + using descending_json = nlohmann::basic_json; + using descending_document = nlohmann::basic_json_document; + std::string text = "{"; + for (char key = 'a'; key <= 'p'; ++key) + { + text += (key != 'a' ? ",\"" : "\"") + std::string(1, key) + "\":[" + std::to_string(key) + "]"; + } + text += "}"; + const descending_json j = descending_json::parse(text); + const descending_document d = descending_document::parse(text); + CHECK(d.root() == j); + CHECK(j == d.root()); + descending_json other = j; + other["k"] = 0; + CHECK(d.root() != other); + CHECK(other != d.root()); + } +} diff --git a/tests/src/unit-json_view_dump.cpp b/tests/src/unit-json_view_dump.cpp index be1ec48ad..fc6501f62 100644 --- a/tests/src/unit-json_view_dump.cpp +++ b/tests/src/unit-json_view_dump.cpp @@ -101,6 +101,10 @@ TEST_CASE("json_view dump") CHECK(d.root().dump(0, ' ', false, json_view::number_format::source) == "[\n1.50,\n1E2,\n-0,\n-0.0,\n123456789012345678901234567890,\n18446744073709551615,\n-9223372036854775808,\n0.1,\n1e-7,\n5e-324\n]"); CHECK(d.root().dump(-1, ' ', true, json_view::number_format::source) == "[1.50,1E2,-0,-0.0,123456789012345678901234567890,18446744073709551615,-9223372036854775808,0.1,1e-7,5e-324]"); + CHECK(d.root().dump(0, ' ', true, json_view::number_format::source) == d.root().dump(0, ' ', false, json_view::number_format::source)); + const json_document escaped = json_document::parse("[\"\xC3\xA9\", 1.50, {\"\xF0\x9F\x98\x80\": 1E2}]"); + CHECK(escaped.root().dump(1, ' ', true, json_view::number_format::source) == "[\n \"\\u00e9\",\n 1.50,\n {\n \"\\ud83d\\ude00\": 1E2\n }\n]"); + // float tokens of up to 17 significant digits in every spelling: those // of at most 15 digits are written from their digits, the others // through the conversion; both as dump() writes them @@ -232,6 +236,29 @@ TEST_CASE("json_view dump") CHECK(editable.root().dump() == expected.dump()); } + // tokens of 16 and 17 digits with few digits after the point, and with a value of 0, which are + // converted by the general algorithm; padded, as a token near the end of the text is not read in this way + { + const std::string padding(40, ' '); + for (const char* token : + { + "123456789012.3456", "-123456789012.3456", "1234567890123.456", "12345678901234.56", "123456789012345.6", "1234567890123456.0", + "0.0000000000000000", "-0.0000000000000000", "0.000000000000000", "1234567890123.4567", "99999999999999.999" + }) + { + CAPTURE(token) + const std::string padded = std::string("[") + token + padding + "]"; + const json_document padded_document = json_document::parse(padded); + const json padded_json = json::parse(padded); + CHECK(padded_document.root()[0].get() == padded_json[0].get()); + CHECK(padded_document.root().dump() == padded_json.dump()); + // a float document converts the tokens as well + using json_float = nlohmann::basic_json; + const nlohmann::basic_json_document float_doc = nlohmann::basic_json_document::parse(padded); + CHECK(float_doc.root()[0].get() == json_float::parse(padded)[0].get()); + } + } + // random doubles, written as parse() and dump() would std::mt19937_64 rng(1170); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed) std::string many = "["; diff --git a/tests/src/unit-json_view_dump_escape.cpp b/tests/src/unit-json_view_dump_escape.cpp index 3def62df6..2716f76d4 100644 --- a/tests/src/unit-json_view_dump_escape.cpp +++ b/tests/src/unit-json_view_dump_escape.cpp @@ -214,6 +214,24 @@ TEST_CASE("json_view dump: escaping, nesting, and output buffers") CHECK(d.root()["a"].dump(-1, ' ', true).size() == 2 + (5000 * 6)); } + SECTION("the estimate for a value that is followed by many decoded strings") + { + // (the end of the value in the source is not known when the nodes after it have decoded strings) + std::string text = "[7"; + for (int i = 0; i < 20; ++i) + { + text += R"(,"\n")"; + } + text += ",8]"; + const json_document d = json_document::parse(text); + CHECK(d.root()[0].dump() == "7"); + CHECK(d.root()[1].dump() == R"("\n")"); + CHECK(d.root().dump() == json::parse(text).dump()); + // the same with a container + const json_document nested = json_document::parse(R"([[7],)" + text.substr(3)); + CHECK(nested.root()[0].dump() == "[7]"); + } + SECTION("ensure_ascii with strings of the source, decoded strings, and edited strings") { // plain ASCII, non-ASCII, escaped (decoded), and mixed strings, as values and as keys; @@ -259,6 +277,28 @@ TEST_CASE("json_view dump: escaping, nesting, and output buffers") } } + SECTION("the length of the run that ensure_ascii copies") + { + // (dump() only asks for strings with a byte to escape, so a run that ends the string is checked here) + for (const std::size_t length : + { + std::size_t{0}, std::size_t{1}, std::size_t{15}, std::size_t{16}, std::size_t{17}, std::size_t{31}, std::size_t{32}, std::size_t{48}, std::size_t{50} + }) + { + CAPTURE(length) + // a run of printable characters, followed by a byte that is not copied and more text + std::string text(length, 'a'); + text += "\"abcdefghijklmnopqrstuvwxyz"; + const auto* const bytes = reinterpret_cast(text.data()); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + const auto* const end = bytes + text.size(); + CHECK(nlohmann::detail::view::ascii_copyable_run(bytes, length, end) == length); + CHECK(nlohmann::detail::view::ascii_copyable_run(bytes, length + 1, end) == length); + CHECK(nlohmann::detail::view::ascii_copyable_run(bytes, length + 5, end) == length); + // the end of the buffer right after the run + CHECK(nlohmann::detail::view::ascii_copyable_run(bytes, length, bytes + length) == length); + } + } + SECTION("ensure_ascii with DEL or a non-ASCII character at every position") { // the check whether a string of the source has no byte >= 0x7F reads diff --git a/tests/src/unit-json_view_edit.cpp b/tests/src/unit-json_view_edit.cpp index 2cc3d4701..db6d3054b 100644 --- a/tests/src/unit-json_view_edit.cpp +++ b/tests/src/unit-json_view_edit.cpp @@ -547,6 +547,10 @@ TEST_CASE("json_view edits: errors") CHECK_THROWS_WITH_AS(d.set(json::json_pointer("/a/01"), 1), "[json.exception.parse_error.106] parse error: array index '01' must not begin with '0'", json::parse_error&); CHECK_THROWS_WITH_AS(d.set(root, json_editable_view()), "[json.exception.type_error.302] type must be a value, but is discarded", json::type_error&); CHECK_THROWS_WITH_AS(d.set(root, json::binary({1, 2})), "[json.exception.type_error.319] cannot store a binary value in a json_document", json::type_error&); + CHECK_THROWS_WITH_AS(d.set(root, json(json::value_t::discarded)), "[json.exception.type_error.302] type must be a value, but is discarded", json::type_error&); + // (an index that does not fit the document, in a null: basic_json would pad it with 4 Gi nulls) + CHECK_THROWS_AS(d.set(json::json_pointer("/z/4294967295"), 1), json::out_of_range&); + CHECK(root["z"].is_null()); #if !defined(JSON_NOEXCEPTION) // invalid UTF-8 is rejected when it enters the document, with the error @@ -572,6 +576,19 @@ TEST_CASE("json_view edits: errors") static_cast(other); } +TEST_CASE("json_view edits: a container that was added is replaced by a scalar") +{ + // the view of an element that push_back() added is that of the new value itself, not of + // the slot that links to it: replacing it by a scalar has to find the container it is in + json_editable_document d = json_editable_document::parse(R"({"a": [1]})"); + d.push_back(d.root()["a"], json::array({2, 3})); + d.push_back(d.root()["a"], json::object({{"k", json::array({4, 5})}})); + d.set(d.root()["a"][1], 6); + d.set(d.root()["a"][2]["k"], "s"); + CHECK(d.root().dump() == R"({"a":[1,6,{"k":"s"}]})"); + CHECK(d.root().materialize() == json::parse(R"({"a":[1,6,{"k":"s"}]})")); +} + #if !defined(JSON_NOEXCEPTION) TEST_CASE("json_view edits: the size of a text arena") { diff --git a/tests/src/unit-json_view_from_json.cpp b/tests/src/unit-json_view_from_json.cpp index 824585a56..50c310cb6 100644 --- a/tests/src/unit-json_view_from_json.cpp +++ b/tests/src/unit-json_view_from_json.cpp @@ -277,6 +277,17 @@ TEST_CASE("json_view from_json") CHECK(true); } + SECTION("a map with enum keys from a view that is not an object") + { + // an object is read with the keys as the enum's names; any other + // value is read as an array of pairs, like a map of any other key type + const json_document d = json_document::parse(R"({"pairs": [["red", 1], ["green", 2]], "number": 1})"); + const json j = d.root().materialize(); + const json_view v = d.root(); + CHECK(same>(v, j, "pairs")); + CHECK((v["pairs"].get>() == std::map {{color::red, 1}, {color::green, 2}})); + } + SECTION("std types: the same values as from the materialized value") { const json_document d = json_document::parse(R"({"ints": [3, 1, 2, 1], "strings": ["b", "a", "b"], "pairs": [[1, "x"], [2, "y"]], diff --git a/tests/src/unit-json_view_image.cpp b/tests/src/unit-json_view_image.cpp index 5e392559d..499f85342 100644 --- a/tests/src/unit-json_view_image.cpp +++ b/tests/src/unit-json_view_image.cpp @@ -819,6 +819,13 @@ TEST_CASE("json_view images: errors") static_cast(d); }) == prefix + "too short"); + // an image that is moved into the document is checked as well + CHECK(exception_of([&] + { + const json_document d = json_document::load(std::vector(10, 0)); + static_cast(d); + }) == prefix + "too short"); + std::vector bad = image; bad[0] = 'X'; CHECK(load_result(bad, image_check::full) == prefix + "unknown format"); diff --git a/tests/src/unit-json_view_large.cpp b/tests/src/unit-json_view_large.cpp index 5ad65c812..b5f58386c 100644 --- a/tests/src/unit-json_view_large.cpp +++ b/tests/src/unit-json_view_large.cpp @@ -333,6 +333,62 @@ TEST_CASE("json_view large objects") CHECK(without_collisions.memory_usage() >= with_collisions.memory_usage() + (table_size * sizeof(std::uint32_t))); } + SECTION("a lookup of a missing key stops at the end of the probe window") + { + // Every slot of the first half of the table holds a key, each at its home slot, so + // that the table is built; a key that is not in the object but whose home slot is + // the first slot finds a full window of slots with other keys. + constexpr std::size_t members = 128; + constexpr std::size_t table_size = 256; // the table size for 128 members: the next power of two >= 256 + for (const std::size_t prefix_length : + { + std::size_t{0}, std::size_t{20} + }) + { + CAPTURE(prefix_length) + std::vector keys(members); + std::vector taken(members, false); + std::string missing; + std::size_t found = 0; + for (std::uint64_t counter = 0; found < members || missing.empty(); ++counter) + { + std::string key(prefix_length, 'p'); + std::uint64_t x = counter; + for (std::size_t i = 0; i < 8; ++i, x /= 26) + { + key += static_cast('a' + (x % 26)); + } + const std::size_t slot = nlohmann::detail::conditional_static_cast(nlohmann::detail::view::key_hash(key.data(), key.size())) & (table_size - 1); + if (slot < members && !taken[slot]) + { + taken[slot] = true; + keys[slot] = key; + ++found; + } + else if (slot == 0 && missing.empty()) + { + missing = key; + } + } + std::string text = "{"; + for (std::size_t i = 0; i < members; ++i) + { + text += (i != 0 ? ",\"" : "\"") + keys[i] + "\":" + std::to_string(i); + } + text += "}"; + const json_document d = json_document::parse(text); + const json_view v = d.root(); + for (std::size_t i = 0; i < members; ++i) + { + CAPTURE(i) + CHECK(v[keys[i]].get() == i); + } + CHECK(!v.contains(missing)); + CHECK(v.find(missing) == v.end()); + CHECK(v.count(missing) == 0); + } + } + SECTION("keys that differ in their last bytes are spread over the table") { // The hash must spread the usual families of keys (numbered names, hex diff --git a/tests/src/unit-json_view_positions.cpp b/tests/src/unit-json_view_positions.cpp index d2e951cef..6fa1fa660 100644 --- a/tests/src/unit-json_view_positions.cpp +++ b/tests/src/unit-json_view_positions.cpp @@ -20,6 +20,7 @@ using nlohmann::ordered_json; using nlohmann::json_document; using nlohmann::json_editable_document; +#include #include #include #include @@ -448,6 +449,31 @@ TEST_CASE("json_view materialize() source positions") CHECK(same_positions(m[std::string("a") + '\n'][2], j[std::string("a") + '\n'][2])); } + SECTION("a damaged text: the end of a container is not found") + { + // (the text of an image is not checked with image_check::none) + const std::string array_text = "[1,2]"; + const std::vector original = json_document::parse(array_text).save(); + const auto at = std::search(original.begin(), original.end(), array_text.begin(), array_text.end()); + REQUIRE(at != original.end()); + const auto closing = static_cast(at - original.begin()) + array_text.size() - 1; + // not a closing bracket, a slash that starts no comment, and the end of the text + for (const char replacement : + { + 'x', '/', ' ' + }) + { + CAPTURE(replacement) + std::vector damaged = original; + damaged[closing] = static_cast(replacement); + const json_document d = json_document::load(damaged, json_document::image_check::none); + const json m = d.root().materialize(); + CHECK(range(m) == "none"); + CHECK(range(m[0]) == "1-2"); + CHECK(range(m[1]) == "3-4"); + } + } + #if !defined(JSON_NOEXCEPTION) SECTION("a decoded string's position must be within the text") { @@ -482,6 +508,7 @@ TEST_CASE("json_view exceptions name the path and the bytes of the value") { "null", "true", "42", "\"s\"", "[]", "{}", "[1, [2, [3, []]]]", R"({"a": {"b": [1, {"c": null}]}, "d": "e"})", R"({"a/b": {"~": [1]}, "\u0041\n": {"x": "\t"}, "": [{}]})", + R"({"a\u0000b": [1, "x"], "c": {"d\u0000": 2}})", // (the path of a key ends at a null byte, as in basic_json) R"([1 /* ] */, {"a": [] // } , "b": {} } , ])", "\n\t[\r\n 1 ,\n {\n \"a\" : [ ]\n }\n]\n", @@ -533,6 +560,49 @@ TEST_CASE("json_view exceptions name the path and the bytes of the value") CHECK(exception_of_call([&] { static_cast(d.root()["a"][2]["n"][0].get()); }) == expected); } + SECTION("views of values that are no longer part of the document, and of no value") + { + const std::string expected_string = "[json.exception.type_error.302] type must be number, but is string"; + static_cast(expected_string); + // (a value that is not found has no path, and with it no bytes) + { + json_editable_document d = json_editable_document::parse(R"(["x", ["y"]])"); + const auto removed = d.root()[0]; + d.erase(d.root(), 0); +#if JSON_DIAGNOSTICS + CHECK(exception_of_call([&] { static_cast(removed.get()); }) == expected_string); +#endif + // (the root is a scalar now: the value cannot be in it) + d.set(d.root(), 5); +#if JSON_DIAGNOSTICS + CHECK(exception_of_call([&] { static_cast(removed.get()); }) == expected_string); +#endif + CHECK(d.root().get() == 5); + } + { + json_editable_document d = json_editable_document::parse(R"(["x"])"); + const auto removed = d.root()[0]; + d.set(d.root(), 5); +#if JSON_DIAGNOSTICS + CHECK(exception_of_call([&] { static_cast(removed.get()); }) == expected_string); +#endif + } + // a view of no value has no context + CHECK(exception_of_call([&] { static_cast(nlohmann::json_view().get()); }) == "[json.exception.type_error.302] type must be number, but is discarded"); + } + + SECTION("an array whose last element was replaced by a new container has no bytes") + { + json_editable_document d = json_editable_document::parse(R"({"a": [1, [2]], "b": 1})"); + d.set(d.root()["a"][1], json::array({3})); + std::string expected = "[json.exception.type_error.302] "; +#if JSON_DIAGNOSTICS + expected += "(/a) "; +#endif + expected += "type must be number, but is array"; + CHECK(exception_of_call([&] { static_cast(d.root()["a"].get()); }) == expected); + } + SECTION("deep documents with siblings after every nested container") { // the search for the path pushes and pops one frame per level (the diff --git a/tests/src/unit-large_json.cpp b/tests/src/unit-large_json.cpp index ed988c242..e8cc0ed42 100644 --- a/tests/src/unit-large_json.cpp +++ b/tests/src/unit-large_json.cpp @@ -12,7 +12,10 @@ using nlohmann::json; #include +#include +#include #include +#include TEST_CASE("tests on very large JSONs") { @@ -140,6 +143,56 @@ TEST_CASE("tests on deeply nested JSONs") } } + SECTION("leaf values of every type below the bound of the recursive descent") + { + // the elements and members of the containers below the bound are + // copied one by one, whatever their type + const json leaves = json::array( + { + nullptr, true, false, -1, std::uint64_t{1}, 1.5, "text", json::binary({1, 2, 3}) + }); + const json members = + { + {"null", nullptr}, {"boolean", true}, {"integer", -1}, {"unsigned", std::uint64_t{1}}, + {"float", 1.5}, {"string", "text"}, {"binary", json::binary({1, 2, 3})} + }; + + for (const bool objects : + { + false, true + }) + { + CAPTURE(objects) + + json j = objects ? members : leaves; + for (std::size_t i = 0; i < 300; ++i) + { + json wrapper = objects ? json::object() : json::array(); + if (objects) + { + wrapper["a"] = std::move(j); + } + else + { + wrapper.push_back(std::move(j)); + } + j = std::move(wrapper); + } + + const json copy(j); // NOLINT(performance-unnecessary-copy-initialization): the copy is what is tested + + const json* current = © + for (std::size_t i = 0; i < 300; ++i) + { + current = objects ? ¤t->at("a") : ¤t->front(); + } + + // the innermost value is shallow: it is safe to compare + CHECK(*current == (objects ? members : leaves)); + CHECK(current->size() == (objects ? members.size() : leaves.size())); + } + } + SECTION("a value that is deep in one place only") { json j = json::object(); diff --git a/tests/src/unit-locale-cpp.cpp b/tests/src/unit-locale-cpp.cpp index b665b6531..441563b3c 100644 --- a/tests/src/unit-locale-cpp.cpp +++ b/tests/src/unit-locale-cpp.cpp @@ -495,3 +495,206 @@ TEST_CASE("locale changes during a single dump() (#5709 item 3)") // fix #5597 made on the parser side CHECK(buf.data == "[\"" + padding + "\",1234.5]"); } + +namespace +{ +// a token whose first data() call switches LC_NUMERIC, so that the locale +// changes between convert_float_locale_aware() looking up the decimal point +// and strtod reading the token (the race of #5198, made deterministic) +struct locale_switching_token +{ + using value_type = char; + + locale_switching_token(std::string text_, const char* switch_to_) + : text(std::move(text_)), switch_to(switch_to_) + {} + + std::size_t size() const + { + return text.size(); + } + + char& operator[](std::size_t i) + { + return text[i]; + } + + char* data() + { + if (!switched) + { + switched = std::setlocale(LC_NUMERIC, switch_to) != nullptr; + } + return &text[0]; + } + + std::string text; + const char* switch_to; + bool switched = false; +}; + +// set LC_NUMERIC to the first of @a names that is usable and has a decimal +// point of @a min_size bytes at least (and at most @a max_size) +bool use_decimal_point_locale(const std::vector& names, std::size_t min_size, std::size_t max_size, const char*& used) +{ + for (const char* name : names) + { + if (std::setlocale(LC_NUMERIC, name) == nullptr) + { + continue; + } + const std::size_t size = std::string(std::localeconv()->decimal_point).size(); + if (size >= min_size && size <= max_size) + { + used = name; + return true; + } + } + return false; +} + +// convert a token with convert_float_locale_aware(); the token must be given +// back as it was +template +FloatType convert_locale_aware(const std::string& text) +{ + std::string token = text; + const std::size_t dot = token.find('.'); + FloatType value{}; + nlohmann::detail::convert_float_locale_aware(token, dot, value); + CHECK(token == text); + return value; +} + +template +void check_locale_aware_conversions() +{ + CHECK(convert_locale_aware("3.5") == FloatType(3.5)); + CHECK(convert_locale_aware("-0.25") == FloatType(-0.25)); + CHECK(convert_locale_aware("1.25e2") == FloatType(125)); + // no decimal point at all + CHECK(convert_locale_aware("12e1") == FloatType(120)); + CHECK(convert_locale_aware("7") == FloatType(7)); +} +} // namespace + +TEST_CASE("convert_float_locale_aware") +{ + const std::vector single_byte_names = {"de_DE.UTF-8", "de_DE", "fr_FR.UTF-8", "fr_FR", "ru_RU.UTF-8"}; + const std::vector multi_byte_names = {"ar_EG.UTF-8", "ar_SA.UTF-8", "fa_IR.UTF-8", "ps_AF.UTF-8", "ar_EG", "fa_IR"}; + const char* used = nullptr; + + SECTION("decimal point '.'") + { + REQUIRE(std::setlocale(LC_NUMERIC, "C") != nullptr); + check_locale_aware_conversions(); + check_locale_aware_conversions(); + check_locale_aware_conversions(); + } + + SECTION("single-byte decimal point other than '.'") + { + if (use_decimal_point_locale(single_byte_names, 1, 1, used) && std::localeconv()->decimal_point[0] != '.') + { + CAPTURE(used) + check_locale_aware_conversions(); + check_locale_aware_conversions(); + check_locale_aware_conversions(); + } + else + { + MESSAGE("no locale with a single-byte decimal point other than '.' is usable"); + } + } + + SECTION("multi-byte decimal point") + { + if (use_decimal_point_locale(multi_byte_names, 2, 8, used)) + { + CAPTURE(used) + check_locale_aware_conversions(); + check_locale_aware_conversions(); + check_locale_aware_conversions(); + } + else + { + MESSAGE("no locale with a multi-byte decimal point is usable"); + } + } + + SECTION("a token that is not converted completely") + { + // the locale does not change, so there is nothing to retry: the + // value up to the garbage is kept + REQUIRE(std::setlocale(LC_NUMERIC, "C") != nullptr); + std::string token = "1.5x"; + double value = 0; + nlohmann::detail::convert_float_locale_aware(token, token.find('.'), value); + CHECK(value == 1.5); + CHECK(token == "1.5x"); + } + + SECTION("the locale changes before the conversion") + { + if (!use_decimal_point_locale(single_byte_names, 1, 1, used) || std::localeconv()->decimal_point[0] == '.') + { + MESSAGE("no locale with a single-byte decimal point other than '.' is usable"); + } + else + { + const std::string comma_locale = used; + REQUIRE(std::setlocale(LC_NUMERIC, "C") != nullptr); + + // "C" -> comma locale: strtod stops at the '.', so the conversion is repeated + { + locale_switching_token token("1.5", comma_locale.c_str()); + double value = 0; + nlohmann::detail::convert_float_locale_aware(token, 1, value); + if (token.switched) + { + CHECK(value == 1.5); + CHECK(token.text == "1.5"); + } + } + + // the same for the other floating-point types + REQUIRE(std::setlocale(LC_NUMERIC, "C") != nullptr); + { + locale_switching_token token("2.25", comma_locale.c_str()); + float value = 0; + nlohmann::detail::convert_float_locale_aware(token, 1, value); + if (token.switched) + { + CHECK(value == 2.25f); + } + } + REQUIRE(std::setlocale(LC_NUMERIC, "C") != nullptr); + { + locale_switching_token token("2.25", comma_locale.c_str()); + long double value = 0; + nlohmann::detail::convert_float_locale_aware(token, 1, value); + if (token.switched) + { + CHECK(value == 2.25L); + } + } + + // comma locale -> multi-byte decimal point + const char* multi_byte = nullptr; + if (use_decimal_point_locale(multi_byte_names, 2, 8, multi_byte)) + { + REQUIRE(std::setlocale(LC_NUMERIC, comma_locale.c_str()) != nullptr); + locale_switching_token token("1.5", multi_byte); + double value = 0; + nlohmann::detail::convert_float_locale_aware(token, 1, value); + if (token.switched) + { + CHECK(value == 1.5); + CHECK(token.text == "1.5"); + } + } + } + } + + CHECK(std::setlocale(LC_NUMERIC, "C") != nullptr); +} diff --git a/tests/src/unit-msgpack-cpp17.cpp b/tests/src/unit-msgpack-cpp17.cpp index 6ae4d6d56..6b6b207bf 100644 --- a/tests/src/unit-msgpack-cpp17.cpp +++ b/tests/src/unit-msgpack-cpp17.cpp @@ -18,6 +18,7 @@ using nlohmann::json; #ifdef JSON_HAS_CPP_17 #include +#include #include // Test suite for verifying MessagePack handling with std::byte input @@ -96,4 +97,33 @@ TEST_CASE("MessagePack with std::byte") } } } + +// JSON text (not MessagePack) as std::byte input: the error position is +// computed from the bytes consumed so far +TEST_CASE("JSON text with std::byte") +{ + const auto to_bytes = [](const std::string & text) + { + std::vector bytes; + for (const char c : text) + { + bytes.push_back(static_cast(c)); + } + return bytes; + }; + + SECTION("valid text") + { + const json j = json::parse(to_bytes("[1,\n2]")); + CHECK(j == json({1, 2})); + CHECK(json::accept(to_bytes("[1,\n2]"))); + } + + SECTION("invalid text") + { + CHECK_FALSE(json::accept(to_bytes("[1,\n2,"))); + CHECK_THROWS_WITH_AS(json::parse(to_bytes("[1,\n2,")), + "[json.exception.parse_error.101] parse error at line 2, column 3: syntax error while parsing value - unexpected end of input; expected '[', '{', or a literal", json::parse_error&); + } +} #endif diff --git a/tests/src/unit-ordered_json.cpp b/tests/src/unit-ordered_json.cpp index c328d8aea..8c07ffa16 100644 --- a/tests/src/unit-ordered_json.cpp +++ b/tests/src/unit-ordered_json.cpp @@ -16,6 +16,7 @@ using nlohmann::ordered_json; #include #include #include +#include TEST_CASE("ordered_json") { @@ -515,3 +516,88 @@ TEST_CASE("parsing random trees of arrays and objects") CHECK(json::parse(text).dump() == expected.dump()); } } + +TEST_CASE("object_key_index") +{ + // the hash table of the DOM parser finds the position of a key in an + // ordered_json object; the parser creates it for exactly 64 elements, but + // it is correct for any number of elements it is given + using nlohmann::detail::object_key_index; + const std::size_t not_found = object_key_index::npos; + + for (const std::size_t initial : + { + std::size_t{0}, std::size_t{10}, std::size_t{64}, std::size_t{65}, std::size_t{300} + }) + { + CAPTURE(initial) + + // the hashes are not distinct on purpose: elements with the same + // hash are told apart by the caller + const auto hash_of = [](std::size_t position) -> std::size_t + { + // truncated to 32 bits where std::size_t is that small + return static_cast((position % 50) * 0x9E3779B97F4A7C15ull); + }; + + std::vector hashes; + for (std::size_t i = 0; i < initial; ++i) + { + hashes.push_back(hash_of(i)); + } + object_key_index index(std::move(hashes)); + + // the elements that are there + for (std::size_t i = 0; i < initial; ++i) + { + CHECK(index.find(hash_of(i), [i](std::size_t position) + { + return position == i; + }) == i); + } + CHECK(index.find(hash_of(1), [](std::size_t /*position*/) + { + return false; + }) == not_found); + + // elements that are added later, until the table grows + for (std::size_t i = initial; i < initial + 600; ++i) + { + index.add(hash_of(i)); + CHECK(index.find(hash_of(i), [i](std::size_t position) + { + return position == i; + }) == i); + } + CHECK(index.find(hash_of(initial), [initial](std::size_t position) + { + return position == initial; + }) == initial); + } +} + +TEST_CASE("JSON pointers on ordered_json") +{ + using pointer = ordered_json::json_pointer; + + SECTION("operator[] inserts a missing key at the end") + { + ordered_json j = {{"z", 1}, {"x", {{"y", 2}}}}; + + j[pointer("/a")] = 3; + j[pointer("/x/b")] = 4; + + CHECK(j == ordered_json({{"z", 1}, {"x", {{"y", 2}, {"b", 4}}}, {"a", 3}})); + CHECK(j.dump() == R"({"z":1,"x":{"y":2,"b":4},"a":3})"); + } + + SECTION("unflatten of a flattened object whose keys are not sorted") + { + // the prefixes /a and /c denote arrays; a key of /a follows one of /c + const ordered_json flat = {{"/a/0/p", 1}, {"/c/0/x", 2}, {"/a/0/q", 3}, {"/a/1/p", 4}}; + const ordered_json expected = {{"a", {{{"p", 1}, {"q", 3}}, {{"p", 4}}}}, {"c", {{{"x", 2}}}}}; + + CHECK(flat.unflatten() == expected); + CHECK(expected.flatten().unflatten() == expected); + } +} diff --git a/tests/src/unit-regression1.cpp b/tests/src/unit-regression1.cpp index b98f2b6f3..53f501f07 100644 --- a/tests/src/unit-regression1.cpp +++ b/tests/src/unit-regression1.cpp @@ -23,11 +23,13 @@ using nlohmann::json; using namespace nlohmann::literals; // NOLINT(google-build-using-namespace) #endif +#include #include #include #include #include #include +#include #include "make_test_data_available.hpp" #include "test_utils.hpp" @@ -1519,6 +1521,76 @@ TEST_CASE("regression tests 1") } } + SECTION("operator[](json_pointer) continues with the member that emplace() found") + { + // see the section above: the lookup does not find "k", but emplace() + // finds "x" and the pointer continues with the member of "x". Here + // the member is a container, so the pointer can go on below it. Only + // the outermost object may be a copy (with restarted timestamps): the + // value of "x" is moved in, so that its own lookups work. + const my_json orig = my_json::parse(R"({"x":null})"); + const auto make = [&orig]() + { + my_json::object_t top = orig.get_ref(); + top.at("x") = my_json::parse(R"({"a":{"b":1},"n":null,"arr":[10,20],"objs":[{"m":5},{"m":6}]})"); + return my_json(std::move(top)); + }; + + SECTION("existing members") + { + // emplace() registers the key "k" in the object, so that every + // pointer needs a new object to reach the member of "x" again + const std::array, 4> pointers = {{{"/k/a/b", "1"}, {"/k/arr/1", "20"}, {"/k/n", "null"}, {"/k/objs/1/m", "6"}}}; + for (const auto& pointer : pointers) + { + CAPTURE(pointer.first) + my_json j = make(); + CHECK(j[my_json::json_pointer(pointer.first)].dump() == pointer.second); + CHECK(j.size() == 1); + CHECK(j["x"].size() == 4); + } + } + + SECTION("a missing member of an object") + { + my_json j = make(); +#ifdef JSON_NO_THREAD_LOCAL + // (set by the user, or by the library for clang with MinGW) + // the objects below the outermost one are not copied by their + // object_t here, but member by member, which keeps their + // timestamps correct: "c" is not mistaken for "b" and is inserted + CHECK(j[my_json::json_pointer("/k/a/c/d")].is_null()); + CHECK(j["x"]["a"].dump() == R"({"b":1,"c":{"d":null}})"); +#else + // the timestamp of "c" is that of "b" again, so that emplace() + // finds the member "b", a number, which has no member "d" + CHECK_THROWS_AS(j[my_json::json_pointer("/k/a/c/d")], my_json::out_of_range&); + CHECK(j["x"]["a"].dump() == R"({"b":1})"); +#endif + } + + SECTION("a null value is converted to an object") + { + my_json j = make(); + j[my_json::json_pointer("/k/n/c")] = 3; + CHECK(j["x"]["n"].dump() == R"({"c":3})"); + } + + SECTION("the end of an array") + { + my_json j = make(); + j[my_json::json_pointer("/k/arr/-")] = 30; + CHECK(j["x"]["arr"].dump() == "[10,20,30]"); + } + + SECTION("an index past the end of an array") + { + my_json j = make(); + j[my_json::json_pointer("/k/arr/3")] = 40; + CHECK(j["x"]["arr"].dump() == "[10,20,null,40]"); + } + } + SECTION("issue #977 - Assigning between different json types") { foo_json lj = ns::foo{3}; diff --git a/tests/src/unit-regression2.cpp b/tests/src/unit-regression2.cpp index c7566c799..08d342d0a 100644 --- a/tests/src/unit-regression2.cpp +++ b/tests/src/unit-regression2.cpp @@ -441,6 +441,10 @@ TEST_CASE("regression tests 2") float_json j2 = {1000.0, 2000.0, 3000.0}; CHECK(float_json::from_ubjson(float_json::to_ubjson(j2, true, true)) == j2); + + // half-precision floats of both signs are converted to the float type + CHECK(float_json::from_cbor(std::vector {0xF9, 0xC0, 0x00}) == -2.0f); + CHECK(float_json::from_cbor(std::vector {0xF9, 0x3C, 0x00}) == 1.0f); } SECTION("issue #1045 - Using STL algorithms with JSON containers with expected results?") diff --git a/tests/src/unit-stream-bulk-scan.cpp b/tests/src/unit-stream-bulk-scan.cpp index d61ce4bd0..0e0fbd72e 100644 --- a/tests/src/unit-stream-bulk-scan.cpp +++ b/tests/src/unit-stream-bulk-scan.cpp @@ -19,11 +19,13 @@ using nlohmann::json; #include +#include #include #include #include #include #include +#include #include #include #include @@ -93,6 +95,33 @@ class chunked_streambuf : public std::streambuf std::size_t m_unbuffered; }; +#if TEST_STREAM_BULK_SCAN +// A streambuf that claims a get area of several gigabytes, of which only the +// first bytes exist. It is only used to move the read position of the adapter, +// never to read beyond them, so that skipping more than gbump() can bump at +// once can be tested without allocating that much. +class oversized_get_area_streambuf : public std::streambuf +{ + public: + explicit oversized_get_area_streambuf(std::size_t claimed_size) + { + // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic) + setg(m_data.data(), m_data.data(), m_data.data() + claimed_size); + } + + // Give the get area back its real size, so that the adapter may read the + // character at the read position when it is destroyed (with + // JSON_PRECISE_STREAM_POSITION, it consumes the last character returned). + void restore_get_area() + { + setg(m_data.data(), m_data.data(), m_data.data() + m_data.size()); + } + + private: + std::array m_data{{'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'}}; +}; +#endif + // how a test reads from the stream enum class read_mode { @@ -433,6 +462,35 @@ TEST_CASE("parsing from a std::istream scans the get area in bulk") ia.copy_token(token, false); CHECK(token.empty()); } + + SECTION("skipping more characters than gbump() takes at once") + { + // gbump() takes an int, but a get area can be larger + const std::size_t claimed = sizeof(std::size_t) < 8 ? 0 : std::size_t{3} << 30u; + oversized_get_area_streambuf buf(claimed); + if (claimed == 0 || static_cast(buf.in_avail()) != claimed) + { + // a 32-bit size_t, or a standard library whose streambuf stores + // the size of the get area in an int (MSVC) + MESSAGE("a get area larger than an int is not supported"); + } + else + { + std::istream is(&buf); + nlohmann::detail::input_stream_adapter ia(is); + CHECK(ia.get_character() == 'a'); + CHECK(ia.bulk_remaining() == claimed - 1); + + const std::size_t skipped = static_cast((std::numeric_limits::max)()) + 10; + ia.bulk_skip(skipped); + CHECK(ia.bulk_remaining() == claimed - 1 - skipped); + + ia.bulk_skip(ia.bulk_remaining()); + CHECK(ia.bulk_remaining() == 0); + + buf.restore_get_area(); + } + } #endif SECTION("SAX parsing from a stream") diff --git a/tests/src/unit-to_chars.cpp b/tests/src/unit-to_chars.cpp index f8863a5a9..3a5a8841d 100644 --- a/tests/src/unit-to_chars.cpp +++ b/tests/src/unit-to_chars.cpp @@ -928,3 +928,49 @@ TEST_CASE("short decimals") } } } + +TEST_CASE("decimals of up to 17 digits") +{ + // write_decimal() lays out digits * 10^exp as format_buffer() does; to_chars() + // calls it for the doubles whose digits write_shortest() does not convert at + // once, and the digits may be as many as 17 + const auto written = [](std::uint64_t digits, int exp) -> std::string + { + std::array buf{}; // (up to 41 bytes are written) + char* const end = nlohmann::detail::dtoa_impl::write_decimal(buf.data(), digits, exp); + return {buf.data(), end}; + }; + + SECTION("fixed notation") + { + CHECK(written(1, 0) == "1.0"); + CHECK(written(1234, -2) == "12.34"); + CHECK(written(123, 2) == "12300.0"); + CHECK(written(50000000000u, -1) == "5000000000.0"); + CHECK(written(100000000u, -8) == "1.0"); + CHECK(written(12345678u, -12) == "1.2345678e-05"); + CHECK(written(1234, 11) == "123400000000000.0"); + // the point moves left of the first digit + CHECK(written(5, -3) == "0.005"); + CHECK(written(12345678901234567u, -20) == "0.00012345678901234567"); + } + + SECTION("exponent notation") + { + CHECK(written(5, -5) == "5e-05"); + CHECK(written(1234, 12) == "1.234e+15"); + CHECK(written(5, -324) == "5e-324"); + } + + SECTION("17 digits") + { + CHECK(written(12345678901234567u, -16) == "1.2345678901234567"); + CHECK(written(12345678901234567u, -19) == "0.0012345678901234567"); + CHECK(written(12345678901234567u, 5) == "1.2345678901234567e+21"); + CHECK(written(99999999999999999u, -17) == "0.99999999999999999"); + CHECK(written(10000000000000001u, -16) == "1.0000000000000001"); + // trailing zeros are not written + CHECK(written(10000000000000000u, -17) == "0.1"); + CHECK(written(12000000000000000u, -15) == "12.0"); + } +} diff --git a/tests/src/unit-ubjson.cpp b/tests/src/unit-ubjson.cpp index 04983da1c..4bc7f7ac0 100644 --- a/tests/src/unit-ubjson.cpp +++ b/tests/src/unit-ubjson.cpp @@ -1673,6 +1673,31 @@ TEST_CASE("UBJSON") return true; }); CHECK_THROWS_AS(scp2.start_array((std::numeric_limits::max)() - 1), json::out_of_range&); + + // without exceptions, the error is reported to the caller + json l; + nlohmann::detail::json_sax_dom_callback_parser scp3(l, [](int /*unused*/, json::parse_event_t /*unused*/, const json& /*unused*/) noexcept + { + return true; + }, false); + CHECK_FALSE(scp3.start_array((std::numeric_limits::max)() - 1)); + CHECK(scp3.is_errored()); + } + + SECTION("array with a known size, read without exceptions") + { + std::vector const v_ubjson = {'[', '#', 'i', 2, 'i', 1, 'i', 2}; + json j; + nlohmann::detail::json_sax_dom_parser sax(j, false); + CHECK(json::sax_parse(v_ubjson, &sax, json::input_format_t::ubjson)); + CHECK(j == json({1, 2})); + + // the readers reject a size this large before they announce + // it, so it can only reach start_array() directly + json k; + nlohmann::detail::json_sax_dom_parser sax2(k, false); + CHECK_FALSE(sax2.start_array((std::numeric_limits::max)() - 1)); + CHECK(sax2.is_errored()); } } } From 5c4f585ac2eca00962bf49eb4f01c17efd627856 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Mon, 2 Nov 2026 17:54:39 +0100 Subject: [PATCH 31/54] Lexer, parser, stream adapter, SAX: remove checks that invariants make dead (#6004) Branch coverage of the test suite and of 24k mutated inputs showed these conditions as never taken; each is implied by the code around it: - lexer: `next_unget` in get_codepoint_bulk() and scan_literal_bulk(). unget() is only called at the end of scan_number(), in scan_comment() and in skip_bom(); every token scan starts with a plain get() in skip_whitespace() that consumes a pending unget, and nothing between that get() and the bulk helpers ungets (now asserts, as in get_after_string_run()) - lexer: `number_type == value_integer` in convert_integer(): both callers exclude value_float and the type has three values - parser: `!context.empty()` in exception_message(): all callers pass a literal - input_stream_adapter: `refill` in get_character_slow(): the function is only called after get_character() found the get area exhausted, and the variable recomputed that comparison - json_sax_dom_callback_parser: `ref_stack.back()` in key() (the null case returned a few lines earlier, and the callback in between cannot reach ref_stack), `is_structured()` in end_object() (non-null entries are only pushed for objects and arrays), `!key_stack.empty()` in current_key() (a kept object pushed its key before any value) No change in behaviour: a build that aborts on each removed side passes the test suite, the fuzz corpora and 40k random documents through every input adapter in four macro configurations. Signed-off-by: Niels Lohmann --- .../nlohmann/detail/input/input_adapters.hpp | 15 ++---- include/nlohmann/detail/input/json_sax.hpp | 7 ++- include/nlohmann/detail/input/lexer.hpp | 21 ++++---- include/nlohmann/detail/input/parser.hpp | 5 +- single_include/nlohmann/json.hpp | 48 ++++++++----------- 5 files changed, 42 insertions(+), 54 deletions(-) diff --git a/include/nlohmann/detail/input/input_adapters.hpp b/include/nlohmann/detail/input/input_adapters.hpp index e9cdb7e09..1d925588f 100644 --- a/include/nlohmann/detail/input/input_adapters.hpp +++ b/include/nlohmann/detail/input/input_adapters.hpp @@ -545,12 +545,9 @@ class input_stream_adapter std::char_traits::int_type get_character_slow() { #if !JSON_DISABLE_STREAM_BULK_SCAN - // sbumpc() refills the get area if it is exhausted - const bool refill = streambuf_access::get_ptr(sb) == streambuf_access::get_end(sb); - if (JSON_HEDLEY_UNLIKELY(refill)) - { - save_token_segment(); - } + // the get area is exhausted (get_character() handles the other case), + // so sbumpc() refills it + save_token_segment(); #endif auto res = sb->sbumpc(); @@ -563,17 +560,13 @@ class input_stream_adapter is->clear(is->rdstate() | std::ios::eofbit); } #if !JSON_DISABLE_STREAM_BULK_SCAN - else if (JSON_HEDLEY_UNLIKELY(refill)) + else { // the consumed character is at gptr() - 1, unless the streambuf // has no get area and returned it from uflow() const char* const next = streambuf_access::get_ptr(sb); after_refill(res, next != nullptr && next != streambuf_access::get_begin(sb)); } - else - { - current_in_get_area = true; // LCOV_EXCL_LINE (called only when the get area is exhausted) - } #endif return res; } diff --git a/include/nlohmann/detail/input/json_sax.hpp b/include/nlohmann/detail/input/json_sax.hpp index 7c6a947a7..43af2616a 100644 --- a/include/nlohmann/detail/input/json_sax.hpp +++ b/include/nlohmann/detail/input/json_sax.hpp @@ -1737,7 +1737,7 @@ class json_sax_dom_callback_parser : public json_sax_dom_error_state key_stack.push_back(val); // add discarded value at the given key and store the reference for later - if (keep && ref_stack.back()) + if (keep) { auto& obj = *ref_stack.back()->m_data.m_value.object; // a single lookup: insert the placeholder, or find the existing member @@ -1794,7 +1794,7 @@ class json_sax_dom_callback_parser : public json_sax_dom_error_state const string_t object_key = pop_container(); - if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_structured()) + if (!ref_stack.empty() && ref_stack.back()) { // remove discarded value remove_discarded_value(*ref_stack.back(), object_key); @@ -1960,8 +1960,7 @@ class json_sax_dom_callback_parser : public json_sax_dom_error_state */ string_t current_key() const { - if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_object() - && !key_stack.empty()) + if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_object()) { return key_stack.back(); } diff --git a/include/nlohmann/detail/input/lexer.hpp b/include/nlohmann/detail/input/lexer.hpp index 7811c9951..1e1d4e8af 100644 --- a/include/nlohmann/detail/input/lexer.hpp +++ b/include/nlohmann/detail/input/lexer.hpp @@ -300,14 +300,16 @@ class lexer : public lexer_base /// calls would (including the position and token bookkeeping of a /// streaming adapter, see bulk_consume) and leaves @a current holding the last of /// the 4 digits, just as the last such get() would; the codepoint is - /// written to @a out. Makes no state change and returns false for a - /// pending unget, fewer than 4 remaining bytes, or any of the 4 bytes not - /// being a hex digit, so the caller falls back unchanged to the - /// per-character loop, which then reports the same diagnostic (stopping - /// at the first invalid digit) as before this optimization. + /// written to @a out. Makes no state change and returns false for fewer + /// than 4 remaining bytes or any of the 4 bytes not being a hex digit, so + /// the caller falls back unchanged to the per-character loop, which then + /// reports the same diagnostic (stopping at the first invalid digit) as + /// before this optimization. bool get_codepoint_bulk(std::true_type /*bulk*/, int& out) { - if (next_unget || ia.bulk_remaining() < 4) + // no unget is pending inside a string (see get_after_string_run()) + JSON_ASSERT(!next_unget); + if (ia.bulk_remaining() < 4) { return false; } @@ -1409,7 +1411,7 @@ scan_number_done: return token_type::value_unsigned; } } - else if (number_type == token_type::value_integer) + else // value_integer (the callers handle value_float themselves) { if (parse_integer_signed(first, last, value_integer)) { @@ -1780,7 +1782,10 @@ scan_number_done: /// the error at the right character JSON_HEDLEY_ALWAYS_INLINE bool scan_literal_bulk(const char* rest, const std::size_t length, std::true_type /*bulk*/) { - if (next_unget || ia.bulk_remaining() < length + // called right after scan_after_whitespace() read current (see + // get_after_string_run()), so no unget is pending + JSON_ASSERT(!next_unget); + if (ia.bulk_remaining() < length || std::memcmp(ia.bulk_data(), rest, length) != 0) { return false; diff --git a/include/nlohmann/detail/input/parser.hpp b/include/nlohmann/detail/input/parser.hpp index 5e0995b05..1446f4524 100644 --- a/include/nlohmann/detail/input/parser.hpp +++ b/include/nlohmann/detail/input/parser.hpp @@ -575,10 +575,7 @@ class parser { std::string error_msg = "syntax error "; - if (!context.empty()) - { - error_msg += concat("while parsing ", context, ' '); - } + error_msg += concat("while parsing ", context, ' '); error_msg += "- "; diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 1818721c1..7fba87d9f 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -9654,12 +9654,9 @@ class input_stream_adapter std::char_traits::int_type get_character_slow() { #if !JSON_DISABLE_STREAM_BULK_SCAN - // sbumpc() refills the get area if it is exhausted - const bool refill = streambuf_access::get_ptr(sb) == streambuf_access::get_end(sb); - if (JSON_HEDLEY_UNLIKELY(refill)) - { - save_token_segment(); - } + // the get area is exhausted (get_character() handles the other case), + // so sbumpc() refills it + save_token_segment(); #endif auto res = sb->sbumpc(); @@ -9672,17 +9669,13 @@ class input_stream_adapter is->clear(is->rdstate() | std::ios::eofbit); } #if !JSON_DISABLE_STREAM_BULK_SCAN - else if (JSON_HEDLEY_UNLIKELY(refill)) + else { // the consumed character is at gptr() - 1, unless the streambuf // has no get area and returned it from uflow() const char* const next = streambuf_access::get_ptr(sb); after_refill(res, next != nullptr && next != streambuf_access::get_begin(sb)); } - else - { - current_in_get_area = true; // LCOV_EXCL_LINE (called only when the get area is exhausted) - } #endif return res; } @@ -13336,14 +13329,16 @@ class lexer : public lexer_base /// calls would (including the position and token bookkeeping of a /// streaming adapter, see bulk_consume) and leaves @a current holding the last of /// the 4 digits, just as the last such get() would; the codepoint is - /// written to @a out. Makes no state change and returns false for a - /// pending unget, fewer than 4 remaining bytes, or any of the 4 bytes not - /// being a hex digit, so the caller falls back unchanged to the - /// per-character loop, which then reports the same diagnostic (stopping - /// at the first invalid digit) as before this optimization. + /// written to @a out. Makes no state change and returns false for fewer + /// than 4 remaining bytes or any of the 4 bytes not being a hex digit, so + /// the caller falls back unchanged to the per-character loop, which then + /// reports the same diagnostic (stopping at the first invalid digit) as + /// before this optimization. bool get_codepoint_bulk(std::true_type /*bulk*/, int& out) { - if (next_unget || ia.bulk_remaining() < 4) + // no unget is pending inside a string (see get_after_string_run()) + JSON_ASSERT(!next_unget); + if (ia.bulk_remaining() < 4) { return false; } @@ -14445,7 +14440,7 @@ scan_number_done: return token_type::value_unsigned; } } - else if (number_type == token_type::value_integer) + else // value_integer (the callers handle value_float themselves) { if (parse_integer_signed(first, last, value_integer)) { @@ -14816,7 +14811,10 @@ scan_number_done: /// the error at the right character JSON_HEDLEY_ALWAYS_INLINE bool scan_literal_bulk(const char* rest, const std::size_t length, std::true_type /*bulk*/) { - if (next_unget || ia.bulk_remaining() < length + // called right after scan_after_whitespace() read current (see + // get_after_string_run()), so no unget is pending + JSON_ASSERT(!next_unget); + if (ia.bulk_remaining() < length || std::memcmp(ia.bulk_data(), rest, length) != 0) { return false; @@ -17373,7 +17371,7 @@ class json_sax_dom_callback_parser : public json_sax_dom_error_state key_stack.push_back(val); // add discarded value at the given key and store the reference for later - if (keep && ref_stack.back()) + if (keep) { auto& obj = *ref_stack.back()->m_data.m_value.object; // a single lookup: insert the placeholder, or find the existing member @@ -17430,7 +17428,7 @@ class json_sax_dom_callback_parser : public json_sax_dom_error_state const string_t object_key = pop_container(); - if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_structured()) + if (!ref_stack.empty() && ref_stack.back()) { // remove discarded value remove_discarded_value(*ref_stack.back(), object_key); @@ -17596,8 +17594,7 @@ class json_sax_dom_callback_parser : public json_sax_dom_error_state */ string_t current_key() const { - if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_object() - && !key_stack.empty()) + if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_object()) { return key_stack.back(); } @@ -23779,10 +23776,7 @@ class parser { std::string error_msg = "syntax error "; - if (!context.empty()) - { - error_msg += concat("while parsing ", context, ' '); - } + error_msg += concat("while parsing ", context, ' '); error_msg += "- "; From 596028c46099801a9b8c99bd06fceae6288222cc Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Mon, 2 Nov 2026 17:54:39 +0100 Subject: [PATCH 32/54] Drop the null_result parameter JSON_IMPLEMENT_OPERATOR no longer uses (#6005) Without the same-type scalar cases, the macro body has no use for it, and JSON_IMPLEMENT_SCALAR_CASES is now used only by JSON_IMPLEMENT_SCALAR_FAST. Signed-off-by: Niels Lohmann --- include/nlohmann/json.hpp | 10 ++++------ single_include/nlohmann/json.hpp | 10 ++++------ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 0447f8823..33ea53a08 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -5388,8 +5388,7 @@ public: /// @name lexicographical comparison operators /// @{ - // the same-type cases for non-owning scalars, shared by JSON_IMPLEMENT_OPERATOR - // and JSON_IMPLEMENT_SCALAR_FAST + // the same-type cases for non-owning scalars, used by JSON_IMPLEMENT_SCALAR_FAST #define JSON_IMPLEMENT_SCALAR_CASES(op, null_result) \ case value_t::null: \ return (null_result); \ @@ -5411,7 +5410,7 @@ public: // because any negative signed value is smaller than any unsigned value. // Otherwise, the non-negative signed value is cast to unsigned before the // comparison to avoid wraparound. -#define JSON_IMPLEMENT_OPERATOR(op, null_result, unordered_result, default_result, deep_result, may_descend) \ +#define JSON_IMPLEMENT_OPERATOR(op, unordered_result, default_result, deep_result, may_descend) \ const auto lhs_type = lhs.type(); \ const auto rhs_type = rhs.type(); \ \ @@ -5548,7 +5547,7 @@ public: JSON_HEDLEY_DIAGNOSTIC_PUSH JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") #endif - JSON_IMPLEMENT_OPERATOR( ==, true, false, false, + JSON_IMPLEMENT_OPERATOR( ==, false, false, compare_iteratively(lhs, rhs, false) == compare_result::equal, true) #ifdef __GNUC__ JSON_HEDLEY_DIAGNOSTIC_POP @@ -5562,7 +5561,6 @@ public: // default_result is used if we cannot compare values. In that case, // we compare types. JSON_IMPLEMENT_OPERATOR(<=>, // *NOPAD* - std::partial_ordering::equivalent, std::partial_ordering::unordered, lhs_type <=> rhs_type, // *NOPAD* to_partial_ordering(compare_iteratively(lhs, rhs, false)), true) @@ -5574,7 +5572,7 @@ public: // default_result is used if we cannot compare values. In that case, // we compare types. Note we have to call the operator explicitly, // because MSVC has problems otherwise. - JSON_IMPLEMENT_OPERATOR( <, false, false, operator<(lhs_type, rhs_type), + JSON_IMPLEMENT_OPERATOR( <, false, operator<(lhs_type, rhs_type), compare_iteratively(lhs, rhs, true) == compare_result::less, false) } #endif diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 7fba87d9f..d953b047f 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -43156,8 +43156,7 @@ public: /// @name lexicographical comparison operators /// @{ - // the same-type cases for non-owning scalars, shared by JSON_IMPLEMENT_OPERATOR - // and JSON_IMPLEMENT_SCALAR_FAST + // the same-type cases for non-owning scalars, used by JSON_IMPLEMENT_SCALAR_FAST #define JSON_IMPLEMENT_SCALAR_CASES(op, null_result) \ case value_t::null: \ return (null_result); \ @@ -43179,7 +43178,7 @@ public: // because any negative signed value is smaller than any unsigned value. // Otherwise, the non-negative signed value is cast to unsigned before the // comparison to avoid wraparound. -#define JSON_IMPLEMENT_OPERATOR(op, null_result, unordered_result, default_result, deep_result, may_descend) \ +#define JSON_IMPLEMENT_OPERATOR(op, unordered_result, default_result, deep_result, may_descend) \ const auto lhs_type = lhs.type(); \ const auto rhs_type = rhs.type(); \ \ @@ -43316,7 +43315,7 @@ public: JSON_HEDLEY_DIAGNOSTIC_PUSH JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") #endif - JSON_IMPLEMENT_OPERATOR( ==, true, false, false, + JSON_IMPLEMENT_OPERATOR( ==, false, false, compare_iteratively(lhs, rhs, false) == compare_result::equal, true) #ifdef __GNUC__ JSON_HEDLEY_DIAGNOSTIC_POP @@ -43330,7 +43329,6 @@ public: // default_result is used if we cannot compare values. In that case, // we compare types. JSON_IMPLEMENT_OPERATOR(<=>, // *NOPAD* - std::partial_ordering::equivalent, std::partial_ordering::unordered, lhs_type <=> rhs_type, // *NOPAD* to_partial_ordering(compare_iteratively(lhs, rhs, false)), true) @@ -43342,7 +43340,7 @@ public: // default_result is used if we cannot compare values. In that case, // we compare types. Note we have to call the operator explicitly, // because MSVC has problems otherwise. - JSON_IMPLEMENT_OPERATOR( <, false, false, operator<(lhs_type, rhs_type), + JSON_IMPLEMENT_OPERATOR( <, false, operator<(lhs_type, rhs_type), compare_iteratively(lhs, rhs, true) == compare_result::less, false) } #endif From 3354436ec46e26095688c1411255268ed896434c Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Mon, 2 Nov 2026 17:54:40 +0100 Subject: [PATCH 33/54] json_view: remove checks that invariants make dead (#6006) Branch coverage of the test suite and of 24k mutated inputs showed these conditions as never taken; each is implied by the code around it: - view_serializer::write_codepoint(): the only caller passes the byte a run of plain output stopped at (a quote, a backslash or a control character), so the function is now write_escaped(), which writes that byte's escape sequence - load_image(): `owned_image.empty() || image != owned_image.data()`; both callers hand in a fresh document_data, and for load(vector&&) the image is owned_image itself - set_moved(): reserve_moved(); all three callers reserved first or have the moved flag set (the documented precondition) - view_side::string() and json_side::string() in equal(): only used after both sides were found to be strings, so they read the string directly (and are noexcept) - number(): the '1'..'9' test after a sign-less dispatch, which only happens for a digit - value_range(): the pre-loop `is_new || off >= size` test, which the loop's first pass repeats on the same node No change in behaviour: a build that aborts on each removed side passes the test suite and the fuzz corpora. Signed-off-by: Niels Lohmann --- include/nlohmann/detail/view/builder.hpp | 3 +- include/nlohmann/detail/view/compare.hpp | 8 ++--- include/nlohmann/detail/view/edit_storage.hpp | 2 +- include/nlohmann/detail/view/errors.hpp | 4 --- include/nlohmann/detail/view/image.hpp | 4 --- include/nlohmann/detail/view/serializer.hpp | 14 +++----- single_include/nlohmann/json_view.hpp | 35 +++++++------------ 7 files changed, 24 insertions(+), 46 deletions(-) diff --git a/include/nlohmann/detail/view/builder.hpp b/include/nlohmann/detail/view/builder.hpp index 938d6d73b..283cae075 100644 --- a/include/nlohmann/detail/view/builder.hpp +++ b/include/nlohmann/detail/view/builder.hpp @@ -1174,8 +1174,9 @@ indent_done: { ++p; } - else if (NLOHMANN_VIEW_LIKELY(cur() >= '1' && cur() <= '9')) + else if (!negative || NLOHMANN_VIEW_LIKELY(cur() >= '1' && cur() <= '9')) { + // (without a sign, the dispatch saw a digit) p = skip_digits(p + 1, e); } else diff --git a/include/nlohmann/detail/view/compare.hpp b/include/nlohmann/detail/view/compare.hpp index 239871f01..2d2c238b9 100644 --- a/include/nlohmann/detail/view/compare.hpp +++ b/include/nlohmann/detail/view/compare.hpp @@ -397,9 +397,9 @@ class view_side return m_view.size(); } - string_view_t string() const + string_view_t string() const noexcept { - return m_view.get_string(); + return {m_view.m_doc->str(*m_view.m_node), m_view.m_node->len}; } bool boolean() const noexcept @@ -522,9 +522,9 @@ class json_side return m_json->size(); } - string_view_t string() const + string_view_t string() const noexcept { - const auto& s = m_json->template get_ref(); + const auto& s = *m_json->template get_ptr(); return string_view_t(s.data(), s.size()); } diff --git a/include/nlohmann/detail/view/edit_storage.hpp b/include/nlohmann/detail/view/edit_storage.hpp index 10898f99f..34b804c46 100644 --- a/include/nlohmann/detail/view/edit_storage.hpp +++ b/include/nlohmann/detail/view/edit_storage.hpp @@ -158,7 +158,7 @@ inline void set_moved(document_data& d, node* n, node* seq, std::size_t cap) e.moved_cap[n->off] = cap; return; } - reserve_moved(d); + // every caller has called reserve_moved() (see above), so neither push_back() can throw e.moved.push_back(seq); e.moved_cap.push_back(cap); n->off = static_cast(e.moved.size() - 1); diff --git a/include/nlohmann/detail/view/errors.hpp b/include/nlohmann/detail/view/errors.hpp index de35d6a51..3c97cef5c 100644 --- a/include/nlohmann/detail/view/errors.hpp +++ b/include/nlohmann/detail/view/errors.hpp @@ -185,10 +185,6 @@ void value_range(const document_data& d, const node* n, std::size_t& start, std: end = p.end; return; } - if ((n->flags & node_flags::is_new) != 0 || n->off >= d.size) - { - return; - } // the open brackets: n, then the last element as long as it is a non-empty container std::size_t closing = 0; std::size_t pos = std::string::npos; diff --git a/include/nlohmann/detail/view/image.hpp b/include/nlohmann/detail/view/image.hpp index 9e57082de..e6680730f 100644 --- a/include/nlohmann/detail/view/image.hpp +++ b/include/nlohmann/detail/view/image.hpp @@ -787,10 +787,6 @@ inline void load_image(document_data& d, const std::uint8_t* image, std::size_t d.edits.reset(); d.base[2] = nullptr; d.owned.clear(); - if (d.owned_image.empty() || image != d.owned_image.data()) - { - d.owned_image.clear(); - } d.arena.clear(); d.indexes.clear(); d.index_slots.clear(); diff --git a/include/nlohmann/detail/view/serializer.hpp b/include/nlohmann/detail/view/serializer.hpp index 44e5a2861..deccec8e4 100644 --- a/include/nlohmann/detail/view/serializer.hpp +++ b/include/nlohmann/detail/view/serializer.hpp @@ -646,7 +646,7 @@ class view_serializer { invalid_utf8(s, n.len); // (the scan stopped at ill-formed UTF-8) } - write_codepoint(s[i], s + i, 1); // a quote, a backslash, or a control character + write_escaped(s[i]); // a quote, a backslash, or a control character ++i; } m_out.put('"'); @@ -1494,8 +1494,9 @@ class view_serializer static_cast(dumped); } // LCOV_EXCL_LINE (dump() throws: the callers pass only ill-formed UTF-8) - template - void write_codepoint(std::uint32_t codepoint, const unsigned char* bytes, std::size_t len) + /// write the escape sequence of a quote, a backslash or a control + /// character (the only bytes a run of plain output stops at) + void write_escaped(std::uint32_t codepoint) { switch (codepoint) { @@ -1523,12 +1524,7 @@ class view_serializer default: break; } - if (codepoint <= 0x1F || (EnsureAscii && codepoint >= 0x7F)) - { - write_u_escape(codepoint); // (a single byte: the only caller passes s[i]) - return; - } - m_out.put(reinterpret_cast(bytes), len); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) LCOV_EXCL_LINE (printable characters are copied in runs) + write_u_escape(codepoint); // (a control character) } void write_u_escape(std::uint32_t u) diff --git a/single_include/nlohmann/json_view.hpp b/single_include/nlohmann/json_view.hpp index bf64662e2..ea04acdbe 100644 --- a/single_include/nlohmann/json_view.hpp +++ b/single_include/nlohmann/json_view.hpp @@ -2479,8 +2479,9 @@ indent_done: { ++p; } - else if (NLOHMANN_VIEW_LIKELY(cur() >= '1' && cur() <= '9')) + else if (!negative || NLOHMANN_VIEW_LIKELY(cur() >= '1' && cur() <= '9')) { + // (without a sign, the dispatch saw a digit) p = skip_digits(p + 1, e); } else @@ -3549,9 +3550,9 @@ class view_side return m_view.size(); } - string_view_t string() const + string_view_t string() const noexcept { - return m_view.get_string(); + return {m_view.m_doc->str(*m_view.m_node), m_view.m_node->len}; } bool boolean() const noexcept @@ -3674,9 +3675,9 @@ class json_side return m_json->size(); } - string_view_t string() const + string_view_t string() const noexcept { - const auto& s = m_json->template get_ref(); + const auto& s = *m_json->template get_ptr(); return string_view_t(s.data(), s.size()); } @@ -5013,10 +5014,6 @@ void value_range(const document_data& d, const node* n, std::size_t& start, std: end = p.end; return; } - if ((n->flags & node_flags::is_new) != 0 || n->off >= d.size) - { - return; - } // the open brackets: n, then the last element as long as it is a non-empty container std::size_t closing = 0; std::size_t pos = std::string::npos; @@ -5475,7 +5472,7 @@ inline void set_moved(document_data& d, node* n, node* seq, std::size_t cap) e.moved_cap[n->off] = cap; return; } - reserve_moved(d); + // every caller has called reserve_moved() (see above), so neither push_back() can throw e.moved.push_back(seq); e.moved_cap.push_back(cap); n->off = static_cast(e.moved.size() - 1); @@ -7779,10 +7776,6 @@ inline void load_image(document_data& d, const std::uint8_t* image, std::size_t d.edits.reset(); d.base[2] = nullptr; d.owned.clear(); - if (d.owned_image.empty() || image != d.owned_image.data()) - { - d.owned_image.clear(); - } d.arena.clear(); d.indexes.clear(); d.index_slots.clear(); @@ -9097,7 +9090,7 @@ class view_serializer { invalid_utf8(s, n.len); // (the scan stopped at ill-formed UTF-8) } - write_codepoint(s[i], s + i, 1); // a quote, a backslash, or a control character + write_escaped(s[i]); // a quote, a backslash, or a control character ++i; } m_out.put('"'); @@ -9945,8 +9938,9 @@ class view_serializer static_cast(dumped); } // LCOV_EXCL_LINE (dump() throws: the callers pass only ill-formed UTF-8) - template - void write_codepoint(std::uint32_t codepoint, const unsigned char* bytes, std::size_t len) + /// write the escape sequence of a quote, a backslash or a control + /// character (the only bytes a run of plain output stops at) + void write_escaped(std::uint32_t codepoint) { switch (codepoint) { @@ -9974,12 +9968,7 @@ class view_serializer default: break; } - if (codepoint <= 0x1F || (EnsureAscii && codepoint >= 0x7F)) - { - write_u_escape(codepoint); // (a single byte: the only caller passes s[i]) - return; - } - m_out.put(reinterpret_cast(bytes), len); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) LCOV_EXCL_LINE (printable characters are copied in runs) + write_u_escape(codepoint); // (a control character) } void write_u_escape(std::uint32_t u) From bd067fd08c8a90fcfc918526adf3c87a306f5abe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Nov 2026 17:57:02 +0100 Subject: [PATCH 34/54] Bump mkdocs-git-revision-date-localized-plugin in /docs/mkdocs (#6011) Bumps [mkdocs-git-revision-date-localized-plugin](https://github.com/timvink/mkdocs-git-revision-date-localized-plugin) from 1.6.0 to 1.6.2. - [Release notes](https://github.com/timvink/mkdocs-git-revision-date-localized-plugin/releases) - [Commits](https://github.com/timvink/mkdocs-git-revision-date-localized-plugin/compare/v1.6.0...v1.6.2) --- updated-dependencies: - dependency-name: mkdocs-git-revision-date-localized-plugin dependency-version: 1.6.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: Niels Lohmann Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- docs/mkdocs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/mkdocs/requirements.txt b/docs/mkdocs/requirements.txt index dc260875f..783f9d88c 100644 --- a/docs/mkdocs/requirements.txt +++ b/docs/mkdocs/requirements.txt @@ -1,7 +1,7 @@ wheel==0.48.0 mkdocs==1.6.1 # documentation framework -mkdocs-git-revision-date-localized-plugin==1.6.0 # plugin "git-revision-date-localized" +mkdocs-git-revision-date-localized-plugin==1.6.2 # plugin "git-revision-date-localized" mkdocs-material==9.8.0 # theme for mkdocs mkdocs-material-extensions==1.3.1 # extensions mkdocs-minify-plugin==0.8.0 # plugin "minify" From 100f676b3e9d85cedbf90e2995cadf9473b6b0e5 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Mon, 2 Nov 2026 18:23:44 +0100 Subject: [PATCH 35/54] Restore the two commits of #6005 that the stack merge dropped (fixes the build of develop) (#6012) * basic_json, iterators, serializer: remove checks that invariants make dead Branch coverage of the test suite and of 24k mutated inputs showed these conditions as never taken; each is implied by the code around it: - copy_object_level(): `src_it != src_object.cend()`; dst was built from exactly src_object.size() unique keys and src_it advances at most once per member - the general comparison operators (equal_general, less_general, three_way_general): the five same-type scalar cases, which the inline fast path of every operator handles before calling them (now an assert); and the NaN half of compares_unordered() in the mixed-type branch, reached only after all six number pairs returned, where only a discarded operand is unordered - serializer, ensure_ascii: the reset of `bytes` under error_handler_t::keep and the surrogate-pair escape in the UTF8_ACCEPT case; every well-formed multi-byte character is escaped by write_u_escapes() before decode() sees it, so decode() only ever accepts DEL here, a single byte of which nothing was buffered (now asserts) - iter_impl(pointer): the if/else chain assigned value-initialized iterators to the value-initialized m_it No change in behaviour: a build that aborts on each removed side passes the test suite, the fuzz corpora and 23.2M dumps of all 1-, 2- and boundary 4-byte strings in both ensure_ascii modes. Signed-off-by: Niels Lohmann * iter_impl: keep the type-dependent initialization in the constructor The check is not needed for correctness (m_it is value-initialized), but without it GCC 13 generates slower iteration loops: a recursive walk over a parsed document takes 0.5-2.7 % longer (callgrind: +1.9 % instructions on twitter.json, +0.95 % on canada.json). Clang generates identical code either way. See the measurements in #6005. Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann --- .../nlohmann/detail/iterators/iter_impl.hpp | 3 ++ include/nlohmann/detail/output/serializer.hpp | 17 +++------ include/nlohmann/json.hpp | 15 +++++--- single_include/nlohmann/json.hpp | 35 ++++++++++--------- 4 files changed, 38 insertions(+), 32 deletions(-) diff --git a/include/nlohmann/detail/iterators/iter_impl.hpp b/include/nlohmann/detail/iterators/iter_impl.hpp index 84a850356..6e6cc0ef6 100644 --- a/include/nlohmann/detail/iterators/iter_impl.hpp +++ b/include/nlohmann/detail/iterators/iter_impl.hpp @@ -108,6 +108,9 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci { JSON_ASSERT(m_object != nullptr); + // m_it is already value-initialized, so this is not needed for + // correctness; it is kept because GCC generates faster loops with it + // (iterating a parsed document is 0.5-2.7 % slower without it, #6005) if (m_object->is_object()) { m_it.object_iterator = typename object_t::iterator(); diff --git a/include/nlohmann/detail/output/serializer.hpp b/include/nlohmann/detail/output/serializer.hpp index fc8e2b384..0c004c491 100644 --- a/include/nlohmann/detail/output/serializer.hpp +++ b/include/nlohmann/detail/output/serializer.hpp @@ -1113,21 +1113,14 @@ class serializer { // the code point is at least 0x60 (smaller ones are handled // above) and is not printable ASCII (which the scanner - // copies): it is DEL (0x7F) under EnsureAscii or a non-ASCII - // character + // copies). Under EnsureAscii it is DEL (0x7F): every + // well-formed multi-byte character is escaped by + // write_u_escapes() above, so decode() only ever accepts a + // single byte here, and nothing of it was buffered if (EnsureAscii) { JSON_ASSERT(codepoint == 0x7F); - if (error_handler == error_handler_t::keep) - { - // this character was buffered as raw bytes below - // in case it turned out to be part of an - // ill-formed sequence (which is kept as is); now - // that it decoded to a well-formed code point, - // undo that and \u-escape it like any other - // character instead - bytes = bytes_after_last_accept; - } + JSON_ASSERT(bytes == bytes_after_last_accept); write_u_escape(bytes, static_cast(codepoint)); } else diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 33ea53a08..9be8a8c6c 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -1579,7 +1579,7 @@ public: auto src_it = src_object.cbegin(); for (auto& element : *dst.m_data.m_value.object) { - if (JSON_HEDLEY_LIKELY(src_it != src_object.cend() && copy_keys_equal(src_it->first, element.first))) + if (JSON_HEDLEY_LIKELY(copy_keys_equal(src_it->first, element.first))) { copy_shallow(src_it->second, element.second, worklist); ++src_it; @@ -5438,8 +5438,15 @@ public: return (*lhs.m_data.m_value.object) op (*rhs.m_data.m_value.object); \ } \ \ - JSON_IMPLEMENT_SCALAR_CASES(op, null_result) \ - \ + /* same-type scalars never get here: the inline fast path of the operator handles them */ \ + case value_t::null: \ + case value_t::boolean: \ + case value_t::number_integer: \ + case value_t::number_unsigned: \ + case value_t::number_float: \ + JSON_ASSERT(false); /* NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) */ \ + return (unordered_result); \ + \ case value_t::string: \ return (*lhs.m_data.m_value.string) op (*rhs.m_data.m_value.string); \ \ @@ -5479,7 +5486,7 @@ public: ? (number_integer_t(-1) op number_integer_t(1)) \ : (static_cast(lhs.m_data.m_value.number_integer) op rhs.m_data.m_value.number_unsigned); \ } \ - else if(compares_unordered(lhs, rhs))\ + else if(lhs_type == value_t::discarded || rhs_type == value_t::discarded)\ {\ return (unordered_result);\ }\ diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index d953b047f..1d7df6d99 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -24104,6 +24104,9 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci { JSON_ASSERT(m_object != nullptr); + // m_it is already value-initialized, so this is not needed for + // correctness; it is kept because GCC generates faster loops with it + // (iterating a parsed document is 0.5-2.7 % slower without it, #6005) if (m_object->is_object()) { m_it.object_iterator = typename object_t::iterator(); @@ -36306,21 +36309,14 @@ class serializer { // the code point is at least 0x60 (smaller ones are handled // above) and is not printable ASCII (which the scanner - // copies): it is DEL (0x7F) under EnsureAscii or a non-ASCII - // character + // copies). Under EnsureAscii it is DEL (0x7F): every + // well-formed multi-byte character is escaped by + // write_u_escapes() above, so decode() only ever accepts a + // single byte here, and nothing of it was buffered if (EnsureAscii) { JSON_ASSERT(codepoint == 0x7F); - if (error_handler == error_handler_t::keep) - { - // this character was buffered as raw bytes below - // in case it turned out to be part of an - // ill-formed sequence (which is kept as is); now - // that it decoded to a well-formed code point, - // undo that and \u-escape it like any other - // character instead - bytes = bytes_after_last_accept; - } + JSON_ASSERT(bytes == bytes_after_last_accept); write_u_escape(bytes, static_cast(codepoint)); } else @@ -39347,7 +39343,7 @@ public: auto src_it = src_object.cbegin(); for (auto& element : *dst.m_data.m_value.object) { - if (JSON_HEDLEY_LIKELY(src_it != src_object.cend() && copy_keys_equal(src_it->first, element.first))) + if (JSON_HEDLEY_LIKELY(copy_keys_equal(src_it->first, element.first))) { copy_shallow(src_it->second, element.second, worklist); ++src_it; @@ -43206,8 +43202,15 @@ public: return (*lhs.m_data.m_value.object) op (*rhs.m_data.m_value.object); \ } \ \ - JSON_IMPLEMENT_SCALAR_CASES(op, null_result) \ - \ + /* same-type scalars never get here: the inline fast path of the operator handles them */ \ + case value_t::null: \ + case value_t::boolean: \ + case value_t::number_integer: \ + case value_t::number_unsigned: \ + case value_t::number_float: \ + JSON_ASSERT(false); /* NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) */ \ + return (unordered_result); \ + \ case value_t::string: \ return (*lhs.m_data.m_value.string) op (*rhs.m_data.m_value.string); \ \ @@ -43247,7 +43250,7 @@ public: ? (number_integer_t(-1) op number_integer_t(1)) \ : (static_cast(lhs.m_data.m_value.number_integer) op rhs.m_data.m_value.number_unsigned); \ } \ - else if(compares_unordered(lhs, rhs))\ + else if(lhs_type == value_t::discarded || rhs_type == value_t::discarded)\ {\ return (unordered_result);\ }\ From 590b622d481d6ec350a42c047bd5fe23baec4a13 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Mon, 2 Nov 2026 18:25:30 +0100 Subject: [PATCH 36/54] parse(): propagate std::bad_alloc instead of crashing on a half-constructed member (#5991) * Create the DOM parser's member value before placing it (#5985) json_sax_dom_parser::set_member() constructed the new value in place over the null member. basic_json(value_t) sets the type before the json_value allocates, so a std::bad_alloc left a member typed object/array with a null pointer, and the destructor crashed instead of propagating the exception. Construct the value first and move it over the member (the move constructor is noexcept). Signed-off-by: Niels Lohmann * Fix CI for the parse() bad_alloc sweep test (#5985) - sweep_allocator: provide max_size(), construct() and destroy(), which GCC 4.8's std::map calls on the allocator directly. - Declare the non-throwing test lambdas noexcept (-Werror=noexcept on recent GCC). - Give the sax_parse lambda an explicit return type; Clang 3.4 crashes on the deduced one in C++11 mode. - Skip the parser callback sweep with iterator debugging (MSVC debug builds, clang-cl), where containers allocate debug proxies inside noexcept move constructors and terminate instead of throwing, as in the other allocator tests. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01N5NasAgbDGG28MqXjQvLqp Signed-off-by: Claude * Construct a scalar member in place; only a throwing value goes through a temporary Creating the value first and moving it over the member is needed only when creating it can throw (a string, a binary value or a container). A scalar or null is again constructed in place, without the extra move and destroy per member (and, with JSON_DIAGNOSTICS, without the extra set_parents()). Signed-off-by: Niels Lohmann * Spell the nothrow trait without && and fix the construct_member comments The dispatch in set_member tests std::is_nothrow_constructible, not whether the value is a scalar. Say so in the comments, and drop the && from the trait argument (equivalent for a forwarding reference) so that astyle no longer inserts spaces around the template brackets. Signed-off-by: Niels Lohmann * Say "for example" in the construct_member comments The overloads are selected by std::is_nothrow_constructible, not by the kind of value: a custom type that cannot throw takes the first one. Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann Signed-off-by: Claude Co-authored-by: Claude --- include/nlohmann/detail/input/json_sax.hpp | 22 ++- single_include/nlohmann/json.hpp | 22 ++- tests/src/unit-allocator.cpp | 172 ++++++++++++++++++++- 3 files changed, 213 insertions(+), 3 deletions(-) diff --git a/include/nlohmann/detail/input/json_sax.hpp b/include/nlohmann/detail/input/json_sax.hpp index 43af2616a..80850ab92 100644 --- a/include/nlohmann/detail/input/json_sax.hpp +++ b/include/nlohmann/detail/input/json_sax.hpp @@ -1303,7 +1303,7 @@ class json_sax_dom_parser : public json_sax_dom_error_state { // a new member is null and holds nothing: construct the value // over it instead of destroying and swapping - ::new (static_cast(&member)) BasicJsonType(std::forward(v)); + construct_member(member, std::forward(v), std::is_nothrow_constructible {}); } else { @@ -1312,6 +1312,26 @@ class json_sax_dom_parser : public json_sax_dom_error_state } } + /// constructing the value cannot throw (for example a number or a boolean): construct it in place + template + JSON_HEDLEY_ALWAYS_INLINE + static void construct_member(BasicJsonType& member, Value&& v, std::true_type /*nothrow*/) + { + ::new (static_cast(&member)) BasicJsonType(std::forward(v)); + } + + /// constructing the value can throw (for example a container or a string), and a + /// constructor that was interrupted must not have touched the member (#5985): + /// create the value first, then move it over the member (the move constructor is + /// noexcept) + template + JSON_HEDLEY_ALWAYS_INLINE + static void construct_member(BasicJsonType& member, Value&& v, std::false_type /*may throw*/) + { + BasicJsonType value(std::forward(v)); + ::new (static_cast(&member)) BasicJsonType(std::move(value)); + } + /// a custom base class may care about the lifetime of the value template static void set_member(BasicJsonType& member, Value&& v, std::false_type /*custom base*/) diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 1d7df6d99..dee9797b4 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -16937,7 +16937,7 @@ class json_sax_dom_parser : public json_sax_dom_error_state { // a new member is null and holds nothing: construct the value // over it instead of destroying and swapping - ::new (static_cast(&member)) BasicJsonType(std::forward(v)); + construct_member(member, std::forward(v), std::is_nothrow_constructible {}); } else { @@ -16946,6 +16946,26 @@ class json_sax_dom_parser : public json_sax_dom_error_state } } + /// constructing the value cannot throw (for example a number or a boolean): construct it in place + template + JSON_HEDLEY_ALWAYS_INLINE + static void construct_member(BasicJsonType& member, Value&& v, std::true_type /*nothrow*/) + { + ::new (static_cast(&member)) BasicJsonType(std::forward(v)); + } + + /// constructing the value can throw (for example a container or a string), and a + /// constructor that was interrupted must not have touched the member (#5985): + /// create the value first, then move it over the member (the move constructor is + /// noexcept) + template + JSON_HEDLEY_ALWAYS_INLINE + static void construct_member(BasicJsonType& member, Value&& v, std::false_type /*may throw*/) + { + BasicJsonType value(std::forward(v)); + ::new (static_cast(&member)) BasicJsonType(std::move(value)); + } + /// a custom base class may care about the lifetime of the value template static void set_member(BasicJsonType& member, Value&& v, std::false_type /*custom base*/) diff --git a/tests/src/unit-allocator.cpp b/tests/src/unit-allocator.cpp index 02b145aae..e1dee5509 100644 --- a/tests/src/unit-allocator.cpp +++ b/tests/src/unit-allocator.cpp @@ -13,6 +13,7 @@ using nlohmann::json; #include +#include #include #if JSON_HAS_RANGES #include @@ -791,7 +792,7 @@ TEST_CASE("a failed allocation leaves the value unchanged") #if JSON_HAS_RANGES && !defined(__MINGW32__) const std::vector numbers = {1, 2}; next_construct_fails = true; - CHECK_THROWS_AS(nlohmann::to_json(j, numbers | std::views::filter([](int /*unused*/) + CHECK_THROWS_AS(nlohmann::to_json(j, numbers | std::views::filter([](int /*unused*/) noexcept { return true; })), std::bad_alloc&); @@ -1024,3 +1025,172 @@ TEST_CASE("insert() provides its documented exception guarantees (#5986)") } } #endif + +#if !defined(JSON_NOEXCEPTION) +namespace +{ +// the allocation numbered sweep_fail_at (counted from 1) throws; 0 never fails +std::size_t sweep_allocations = 0; +std::size_t sweep_fail_at = 0; +std::ptrdiff_t sweep_live = 0; + +template +struct sweep_allocator +{ + using value_type = T; + + sweep_allocator() = default; + template sweep_allocator(const sweep_allocator& /*unused*/) noexcept {} + + T* allocate(std::size_t n) + { + if (sweep_fail_at != 0 && ++sweep_allocations == sweep_fail_at) + { + throw std::bad_alloc(); + } + ++sweep_live; + return std::allocator().allocate(n); + } + + void deallocate(T* p, std::size_t n) noexcept + { + --sweep_live; + std::allocator().deallocate(p, n); + } + + // GCC 4.8's std::map calls these on the allocator directly instead of + // going through std::allocator_traits + std::size_t max_size() const noexcept + { + return (std::numeric_limits::max)() / sizeof(T); + } + + template + void construct(U* p, Args&& ... args) noexcept(noexcept(::new (static_cast(p)) U(std::forward(args)...))) + { + ::new (static_cast(p)) U(std::forward(args)...); + } + + template + void destroy(U* p) noexcept + { + static_cast(p); // unused for trivially destructible types + p->~U(); + } + + template + struct rebind + { + using other = sweep_allocator; + }; + + bool operator==(const sweep_allocator& /*unused*/) const noexcept + { + return true; + } + bool operator!=(const sweep_allocator& /*unused*/) const noexcept + { + return false; + } +}; + +// run make() with the allocation numbered k failing, for k = 1, 2, ...: every +// failure must be a std::bad_alloc and must not leave anything behind, and +// the first k without a failure must yield the value made without failures +template +void sweep_allocations_of(const Make& make) +{ + sweep_fail_at = 0; + const BasicJsonType expected = make(); + const std::ptrdiff_t live_before = sweep_live; + + std::size_t failures = 0; + for (std::size_t k = 1; k < 10000; ++k) + { + sweep_allocations = 0; + sweep_fail_at = k; + try + { + const BasicJsonType result = make(); + sweep_fail_at = 0; + CHECK(result == expected); + break; + } + catch (const std::bad_alloc&) + { + sweep_fail_at = 0; + ++failures; + } + CHECK(sweep_live == live_before); + } + sweep_fail_at = 0; + CHECK(failures > 10); + CHECK(sweep_live == live_before); +} +} // namespace + +TEST_CASE("parse() survives a failing allocation at every point (#5985)") +{ + // strings beyond the small string buffer, nested objects and arrays + const std::string text = R"({"a":"a long string that does not fit into the small string buffer1","b":[1,2,[3,{"k":"another string that is longer than any small string buffer"}]],"c":{"x":1,"y":{"z":[]},"w":"yet another string that does not fit into a small buffer"},"d":1,"b":[[],{}]})"; + + using sweep_json = nlohmann::json::with_allocator_t; + using sweep_ordered_json = nlohmann::ordered_json::with_allocator_t; + + SECTION("json") + { + sweep_allocations_of([&] + { + return sweep_json::parse(text); + }); + } + + SECTION("ordered_json") + { + sweep_allocations_of([&] + { + return sweep_ordered_json::parse(text); + }); + } + + // With iterator debugging (MSVC STL debug builds, also used by clang-cl), + // containers allocate a debug proxy through the allocator inside their + // noexcept move constructors, so failing that allocation terminates the + // program instead of throwing std::bad_alloc. Nothing to check there. +#if !(defined(_ITERATOR_DEBUG_LEVEL) && _ITERATOR_DEBUG_LEVEL > 0) + SECTION("parser callback") + { + const auto keep_all = [](int /*depth*/, sweep_json::parse_event_t /*event*/, sweep_json& /*parsed*/) noexcept + { + return true; + }; + sweep_allocations_of([&] + { + return sweep_json::parse(text, keep_all); + }); + + const auto drop_d = [](int /*depth*/, sweep_json::parse_event_t event, sweep_json & parsed) + { + return event != sweep_json::parse_event_t::key || parsed != "d"; + }; + sweep_allocations_of([&] + { + return sweep_json::parse(text, drop_d); + }); + } +#endif + + SECTION("sax_parse into an existing value") + { + // a failed parse leaves the target in a state that is safe to destroy + using sax_t = nlohmann::detail::json_sax_dom_parser>; + sweep_allocations_of([&]() -> sweep_json + { + sweep_json value; + sax_t sax(value, false); + static_cast(sweep_json::sax_parse(text, &sax)); + return value; + }); + } +} +#endif From 6d5581c4e8dbc10b9281eba6adbb81150f650cab Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Mon, 2 Nov 2026 18:25:44 +0100 Subject: [PATCH 37/54] CBOR/MessagePack readers: leave one-element arrays to the general path (#6008) enter_cbor_array and enter_msgpack_array called the out-of-line scalar run for every non-empty array. When the array's element is an object or an array, as on every level of [[...]] or the single ring of a GeoJSON polygon, the call reads nothing; it made 1,000 arrays nested in each other cost 4 % more instructions (GCC 13) and up to 8 % more time (Apple clang) than without the run. #5975 had fixed that with an inline pre-check, which was later dropped after measuring canada only. The run is now entered only for arrays of two or more elements: the same comparison as before, so no other row changes. Arrays of one scalar, which the run read faster, now take the general path; they are rare in documents (one in citm_catalog, none in twitter or canada). The new benchmark FromBinaryShape/singles measures them, next to FromBinaryShape/nested. Instructions per repetition, new/old, GCC 13 callgrind: nested arrays 0.973 (CBOR) and 0.971 (MessagePack), arrays of one scalar 1.037 and 1.021, every other row within 0.15 %. Signed-off-by: Niels Lohmann --- .../nlohmann/detail/input/binary_reader.hpp | 14 ++++++++-- single_include/nlohmann/json.hpp | 14 ++++++++-- tests/benchmarks/src/benchmarks.cpp | 18 ++++++++++++ tests/src/unit-cbor.cpp | 28 +++++++++++++++++++ tests/src/unit-msgpack.cpp | 28 +++++++++++++++++++ 5 files changed, 96 insertions(+), 6 deletions(-) diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index 8a6a0a6d5..8edc53ae9 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -1856,14 +1856,18 @@ class binary_reader Only the start of an array is tried, which covers arrays of numbers (e.g. coordinates) but costs arrays of other values only one call, not one - per element. + per element. An array of one element is not tried at all: that element is + mostly an object or an array (`[[...]]`, the single ring of a GeoJSON + polygon), for which the call would find no scalar, while a single scalar, + which the run would read faster, is rare. The benchmarks + `FromBinaryShape/nested` and `FromBinaryShape/singles` show both sides. @param[in] len the number of elements @return whether the SAX parser accepted the array and the values read */ bool enter_cbor_array(const std::size_t len) { - return enter_array(len) && (len == 0 || parse_cbor_scalar_run(std::integral_constant {})); + return enter_array(len) && (len <= 1 || parse_cbor_scalar_run(std::integral_constant {})); } /// input that is not contiguous: every value is read by parse_cbor_value @@ -2764,12 +2768,16 @@ class binary_reader /*! @brief open a MessagePack array and read the scalars it begins with (see @ref parse_msgpack_scalar_run) + + An array of one element is not tried, for the reason given at + @ref enter_cbor_array. + @param[in] len the number of elements @return whether the SAX parser accepted the array and the values read */ bool enter_msgpack_array(const std::size_t len) { - return enter_array(len) && (len == 0 || parse_msgpack_scalar_run(std::integral_constant {})); + return enter_array(len) && (len <= 1 || parse_msgpack_scalar_run(std::integral_constant {})); } /// input that is not contiguous: every value is read by parse_msgpack_value diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index dee9797b4..7f58db2af 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -19871,14 +19871,18 @@ class binary_reader Only the start of an array is tried, which covers arrays of numbers (e.g. coordinates) but costs arrays of other values only one call, not one - per element. + per element. An array of one element is not tried at all: that element is + mostly an object or an array (`[[...]]`, the single ring of a GeoJSON + polygon), for which the call would find no scalar, while a single scalar, + which the run would read faster, is rare. The benchmarks + `FromBinaryShape/nested` and `FromBinaryShape/singles` show both sides. @param[in] len the number of elements @return whether the SAX parser accepted the array and the values read */ bool enter_cbor_array(const std::size_t len) { - return enter_array(len) && (len == 0 || parse_cbor_scalar_run(std::integral_constant {})); + return enter_array(len) && (len <= 1 || parse_cbor_scalar_run(std::integral_constant {})); } /// input that is not contiguous: every value is read by parse_cbor_value @@ -20779,12 +20783,16 @@ class binary_reader /*! @brief open a MessagePack array and read the scalars it begins with (see @ref parse_msgpack_scalar_run) + + An array of one element is not tried, for the reason given at + @ref enter_cbor_array. + @param[in] len the number of elements @return whether the SAX parser accepted the array and the values read */ bool enter_msgpack_array(const std::size_t len) { - return enter_array(len) && (len == 0 || parse_msgpack_scalar_run(std::integral_constant {})); + return enter_array(len) && (len <= 1 || parse_msgpack_scalar_run(std::integral_constant {})); } /// input that is not contiguous: every value is read by parse_msgpack_value diff --git a/tests/benchmarks/src/benchmarks.cpp b/tests/benchmarks/src/benchmarks.cpp index 0d9e06d21..9e38a420c 100644 --- a/tests/benchmarks/src/benchmarks.cpp +++ b/tests/benchmarks/src/benchmarks.cpp @@ -629,6 +629,22 @@ static json make_containers() return j; } +/// many arrays of one number each: one-element arrays are left to the general +/// path (CBOR and MessagePack), which reads a lone number slower than the loop +/// for arrays of scalars would +static json make_singles() +{ + json data = json::array(); + for (std::size_t i = 0; i < 100000; ++i) + { + data.push_back(json::array({i})); + } + + json j = json::object(); + j["data"] = std::move(data); + return j; +} + /// one flat array of numbers: the scalar decoding path, which must not move static json make_scalars() { @@ -676,6 +692,8 @@ BENCHMARK_CAPTURE(FromBinaryShape, containers / ubjson_optimized, make_container BENCHMARK_CAPTURE(FromBinaryShape, containers / bjdata, make_containers, binary_format::bjdata); BENCHMARK_CAPTURE(FromBinaryShape, containers / bson, make_containers, binary_format::bson); BENCHMARK_CAPTURE(FromBinaryShape, containers / bon8, make_containers, binary_format::bon8); +BENCHMARK_CAPTURE(FromBinaryShape, singles / cbor, make_singles, binary_format::cbor); +BENCHMARK_CAPTURE(FromBinaryShape, singles / msgpack, make_singles, binary_format::msgpack); // BSON names every array element, so a large array measures key generation // rather than scalar decoding and is left out here BENCHMARK_CAPTURE(FromBinaryShape, scalars / cbor, make_scalars, binary_format::cbor); diff --git a/tests/src/unit-cbor.cpp b/tests/src/unit-cbor.cpp index 442050cf4..c4d29543d 100644 --- a/tests/src/unit-cbor.cpp +++ b/tests/src/unit-cbor.cpp @@ -2544,6 +2544,34 @@ TEST_CASE("CBOR arrays of scalars from contiguous and non-contiguous input") } } +TEST_CASE("CBOR arrays of one element from contiguous and non-contiguous input") +{ + // contiguous input leaves an array of one element to the general path + // rather than to the loop that reads arrays of scalars; both inputs must + // still give the same events, value, or error for the complete document + // and for every truncation of it, with enough bytes after each array that + // the loop could have been entered + const json j = json::parse(R"({"a": [[[[1.5]]]], "b": [7], "c": [{"x": 2.5}], "d": ["s"], "e": [[1, 2]], "f": [null], "g": 1.25})"); + const std::vector bytes = json::to_cbor(j); + + CHECK(json::from_cbor(bytes) == j); + + for (std::size_t length = 0; length <= bytes.size(); ++length) + { + CAPTURE(length) + const std::vector contiguous(bytes.begin(), bytes.begin() + static_cast(length)); + const std::list bytewise(contiguous.begin(), contiguous.end()); + + CHECK(json::from_cbor(contiguous, true, false).dump() == json::from_cbor(bytewise.begin(), bytewise.end(), true, false).dump()); + + SaxEventLogger from_contiguous; + SaxEventLogger from_bytewise; + CHECK(json::sax_parse(contiguous, &from_contiguous, json::input_format_t::cbor, true, false) + == json::sax_parse(bytewise.begin(), bytewise.end(), &from_bytewise, json::input_format_t::cbor, true, false)); + CHECK(from_contiguous.events == from_bytewise.events); + } +} + TEST_CASE("CBOR indefinite-length strings do not recurse per chunk") { // Reading an indefinite-length string or byte array used to call itself diff --git a/tests/src/unit-msgpack.cpp b/tests/src/unit-msgpack.cpp index 9b5891fdd..9c50d22fd 100644 --- a/tests/src/unit-msgpack.cpp +++ b/tests/src/unit-msgpack.cpp @@ -2311,3 +2311,31 @@ TEST_CASE("MessagePack arrays of scalars from contiguous and non-contiguous inpu == json::sax_parse(bytewise.begin(), bytewise.end(), &from_bytewise, json::input_format_t::msgpack)); } } + +TEST_CASE("MessagePack arrays of one element from contiguous and non-contiguous input") +{ + // contiguous input leaves an array of one element to the general path + // rather than to the loop that reads arrays of scalars; both inputs must + // still give the same events, value, or error for the complete document + // and for every truncation of it, with enough bytes after each array that + // the loop could have been entered + const json j = json::parse(R"({"a": [[[[1.5]]]], "b": [7], "c": [{"x": 2.5}], "d": ["s"], "e": [[1, 2]], "f": [null], "g": 1.25})"); + const std::vector bytes = json::to_msgpack(j); + + CHECK(json::from_msgpack(bytes) == j); + + for (std::size_t length = 0; length <= bytes.size(); ++length) + { + CAPTURE(length) + const std::vector contiguous(bytes.begin(), bytes.begin() + static_cast(length)); + const std::list bytewise(contiguous.begin(), contiguous.end()); + + CHECK(json::from_msgpack(contiguous, true, false).dump() == json::from_msgpack(bytewise.begin(), bytewise.end(), true, false).dump()); + + SaxEventLogger from_contiguous; + SaxEventLogger from_bytewise; + CHECK(json::sax_parse(contiguous, &from_contiguous, json::input_format_t::msgpack, true, false) + == json::sax_parse(bytewise.begin(), bytewise.end(), &from_bytewise, json::input_format_t::msgpack, true, false)); + CHECK(from_contiguous.events == from_bytewise.events); + } +} From b18b6fc7e6223075b4447d08323ba7e81728febe Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Mon, 2 Nov 2026 18:27:08 +0100 Subject: [PATCH 38/54] Build the parser's syntax error prefix with one concat() (#6013) Since #6004 made the "while parsing" part unconditional, the three appends at the start of exception_message() are one fixed string around the context. Review follow-up to #6004. Signed-off-by: Niels Lohmann --- include/nlohmann/detail/input/parser.hpp | 6 +----- single_include/nlohmann/json.hpp | 6 +----- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/include/nlohmann/detail/input/parser.hpp b/include/nlohmann/detail/input/parser.hpp index 1446f4524..9335542a0 100644 --- a/include/nlohmann/detail/input/parser.hpp +++ b/include/nlohmann/detail/input/parser.hpp @@ -573,11 +573,7 @@ class parser std::string exception_message(const token_type expected, const std::string& context) { - std::string error_msg = "syntax error "; - - error_msg += concat("while parsing ", context, ' '); - - error_msg += "- "; + std::string error_msg = concat("syntax error while parsing ", context, " - "); if (last_token == token_type::parse_error) { diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 7f58db2af..cf5a4d711 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -23802,11 +23802,7 @@ class parser std::string exception_message(const token_type expected, const std::string& context) { - std::string error_msg = "syntax error "; - - error_msg += concat("while parsing ", context, ' '); - - error_msg += "- "; + std::string error_msg = concat("syntax error while parsing ", context, " - "); if (last_token == token_type::parse_error) { From 07876e22920ccc6770af188f4eb2ba4febb98508 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Mon, 2 Nov 2026 19:11:07 +0100 Subject: [PATCH 39/54] json_view: a value that is not found has no bytes in its exception context (#6015) --- include/nlohmann/detail/view/errors.hpp | 8 ++++++-- single_include/nlohmann/json_view.hpp | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/include/nlohmann/detail/view/errors.hpp b/include/nlohmann/detail/view/errors.hpp index 3c97cef5c..74597edf0 100644 --- a/include/nlohmann/detail/view/errors.hpp +++ b/include/nlohmann/detail/view/errors.hpp @@ -251,8 +251,12 @@ std::string diagnostics(const error_site site) std::string context; #if JSON_DIAGNOSTICS std::vector tokens; - // a value that is not found leaves no tokens, like the root - static_cast(edited ? value_path(d, site.value(), tokens) : value_path(d, site.value(), tokens)); + const bool found = edited ? value_path(d, site.value(), tokens) : value_path(d, site.value(), tokens); + if (!found) + { + // a value that is not found has no path, and with it no bytes + return context; + } if (!tokens.empty()) { // the root value has no path, but still has a byte range diff --git a/single_include/nlohmann/json_view.hpp b/single_include/nlohmann/json_view.hpp index ea04acdbe..1c6cd6cef 100644 --- a/single_include/nlohmann/json_view.hpp +++ b/single_include/nlohmann/json_view.hpp @@ -5080,8 +5080,12 @@ std::string diagnostics(const error_site site) std::string context; #if JSON_DIAGNOSTICS std::vector tokens; - // a value that is not found leaves no tokens, like the root - static_cast(edited ? value_path(d, site.value(), tokens) : value_path(d, site.value(), tokens)); + const bool found = edited ? value_path(d, site.value(), tokens) : value_path(d, site.value(), tokens); + if (!found) + { + // a value that is not found has no path, and with it no bytes + return context; + } if (!tokens.empty()) { // the root value has no path, but still has a byte range From baba5db36f1747556499d70067a60b56d8989c06 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Mon, 2 Nov 2026 20:07:00 +0100 Subject: [PATCH 40/54] Fix the MSVC and GCC builds of develop (C4127, -Weffc++) (#6016) json_view number(): #6006 folded the check of the digit after a minus sign into `else if (!negative || ...)`, which MSVC 19.44 reports as C4127 (conditional expression is constant) and /WX turns into an error in every MSVC and clang-cl job. The check now sits in the `if (negative)` block, right after the sign is skipped; the error and its position are unchanged. unit-locale-cpp.cpp: the locale_switching_token added in #5997 held a `const char*`, so GCC's -Weffc++ asked for a copy constructor and copy assignment. The locale name is now a std::string. Signed-off-by: Niels Lohmann --- include/nlohmann/detail/view/builder.hpp | 12 ++++++------ single_include/nlohmann/json_view.hpp | 12 ++++++------ tests/src/unit-locale-cpp.cpp | 8 ++++---- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/include/nlohmann/detail/view/builder.hpp b/include/nlohmann/detail/view/builder.hpp index 283cae075..14be4ea7c 100644 --- a/include/nlohmann/detail/view/builder.hpp +++ b/include/nlohmann/detail/view/builder.hpp @@ -1168,20 +1168,20 @@ indent_done: if (negative) { ++p; + if (NLOHMANN_VIEW_UNLIKELY(cur() < '0' || cur() > '9')) + { + return fail(error_code::number_after_minus); + } } const unsigned char* const int_start = p; if (cur() == '0') { ++p; } - else if (!negative || NLOHMANN_VIEW_LIKELY(cur() >= '1' && cur() <= '9')) - { - // (without a sign, the dispatch saw a digit) - p = skip_digits(p + 1, e); - } else { - return fail(error_code::number_after_minus); + // (a digit: the dispatch or the check after the sign saw it) + p = skip_digits(p + 1, e); } const auto int_digits = static_cast(p - int_start); std::size_t frac_digits = 0; diff --git a/single_include/nlohmann/json_view.hpp b/single_include/nlohmann/json_view.hpp index 1c6cd6cef..22c72d982 100644 --- a/single_include/nlohmann/json_view.hpp +++ b/single_include/nlohmann/json_view.hpp @@ -2473,20 +2473,20 @@ indent_done: if (negative) { ++p; + if (NLOHMANN_VIEW_UNLIKELY(cur() < '0' || cur() > '9')) + { + return fail(error_code::number_after_minus); + } } const unsigned char* const int_start = p; if (cur() == '0') { ++p; } - else if (!negative || NLOHMANN_VIEW_LIKELY(cur() >= '1' && cur() <= '9')) - { - // (without a sign, the dispatch saw a digit) - p = skip_digits(p + 1, e); - } else { - return fail(error_code::number_after_minus); + // (a digit: the dispatch or the check after the sign saw it) + p = skip_digits(p + 1, e); } const auto int_digits = static_cast(p - int_start); std::size_t frac_digits = 0; diff --git a/tests/src/unit-locale-cpp.cpp b/tests/src/unit-locale-cpp.cpp index 441563b3c..19c08166c 100644 --- a/tests/src/unit-locale-cpp.cpp +++ b/tests/src/unit-locale-cpp.cpp @@ -505,8 +505,8 @@ struct locale_switching_token { using value_type = char; - locale_switching_token(std::string text_, const char* switch_to_) - : text(std::move(text_)), switch_to(switch_to_) + locale_switching_token(std::string text_, std::string switch_to_) + : text(std::move(text_)), switch_to(std::move(switch_to_)) {} std::size_t size() const @@ -523,13 +523,13 @@ struct locale_switching_token { if (!switched) { - switched = std::setlocale(LC_NUMERIC, switch_to) != nullptr; + switched = std::setlocale(LC_NUMERIC, switch_to.c_str()) != nullptr; } return &text[0]; } std::string text; - const char* switch_to; + std::string switch_to; bool switched = false; }; From e32051125faf95dc64d49a129959b6742b9ffee9 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Tue, 3 Nov 2026 13:42:46 +0100 Subject: [PATCH 41/54] Fix the build of develop: tests from #5997 and #5991 (#6019) * Fix the build of develop: tests from #5997 and #5991 - unit-msgpack-cpp17: assign the result of json::parse in CHECK_THROWS_WITH_AS (GCC -Werror=unused-result) - unit-json_view: failing_allocator::allocate is [[noreturn]] (clang -Wmissing-noreturn) - unit-class_lexer: build the quoted string once (clang-tidy performance-inefficient-string-concatenation) - unit-allocator: sweep_allocator provides pointer, reference and size typedefs, which GCC 4.8's std::map takes from the allocator - unit-allocator: skip the "sax_parse into an existing value" sweep with iterator debugging, like the parser callback sweep: MSVC's std::vector default constructor is noexcept but allocates its debug proxy through the allocator, so a failing allocation in create() terminates Signed-off-by: Niels Lohmann * Fix the remaining CI failures of the tests from #5997 and #5991 - unit-custom-object-key-type: drop the redundant member initializer and return a braced init list (clang-tidy); direct-initialize the converted value, as copy-initialization makes GCC 4.8 instantiate the copy constructor, which needs a range constructor strict_object lacks - unit-stream-bulk-scan: pick the claimed get area size by tag dispatch instead of a constant condition (MSVC C4127) - unit-allocator: skip the parser callback and sax_parse sweeps with Clang 3.4 and 3.5, which leak memory when an element constructor throws inside a libstdc++ std::map copy or std::vector reallocation Signed-off-by: Niels Lohmann * Fix the CI failures that the first failing TU of each job hid - unit-stream-bulk-scan: compute the claimed size without a condition; the tag-dispatched template was unused with a 64-bit std::size_t (-Wunused-template) - unit-json_view: give failing_allocator the typedefs, rebind and members that GCC 4.8's std::map takes from the allocator directly - clang-tidy hits in the tests from #5997: raw string literals (unit-json_view), redundant c_str() and a NOLINT for &text[0] (unit-locale-cpp), reserve() before a loop (unit-ordered_json), and a NOLINT for a return that a braced list would turn into an array (unit-regression1) Signed-off-by: Niels Lohmann * Skip the 3 GiB get-area test with a 32-bit std::size_t in the preprocessor The claimed size was a constant 0 on Win32, so `claimed == 0 || ...` was a constant condition and MSVC reported C4127. Guard the section with `#if SIZE_MAX > UINT32_MAX` instead, as unit-msgpack-custom-types.cpp does. Signed-off-by: Niels Lohmann * Silence modernize-use-transparent-functors for failing_map in unit-json_view std::less<> needs C++14, and the test is also built as C++11. Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann --- tests/src/unit-allocator.cpp | 20 ++++++++++-- tests/src/unit-class_lexer.cpp | 7 ++-- tests/src/unit-custom-object-key-type.cpp | 8 +++-- tests/src/unit-json_view.cpp | 40 +++++++++++++++++++++-- tests/src/unit-locale-cpp.cpp | 8 ++--- tests/src/unit-msgpack-cpp17.cpp | 3 +- tests/src/unit-ordered_json.cpp | 1 + tests/src/unit-regression1.cpp | 2 +- tests/src/unit-stream-bulk-scan.cpp | 13 +++++--- 9 files changed, 80 insertions(+), 22 deletions(-) diff --git a/tests/src/unit-allocator.cpp b/tests/src/unit-allocator.cpp index e1dee5509..8fa9034b5 100644 --- a/tests/src/unit-allocator.cpp +++ b/tests/src/unit-allocator.cpp @@ -1038,6 +1038,14 @@ template struct sweep_allocator { using value_type = T; + // GCC 4.8's std::map takes these from the allocator instead of from + // std::allocator_traits + using pointer = T*; + using const_pointer = const T*; + using reference = T&; + using const_reference = const T&; + using size_type = std::size_t; + using difference_type = std::ptrdiff_t; sweep_allocator() = default; template sweep_allocator(const sweep_allocator& /*unused*/) noexcept {} @@ -1155,9 +1163,14 @@ TEST_CASE("parse() survives a failing allocation at every point (#5985)") // With iterator debugging (MSVC STL debug builds, also used by clang-cl), // containers allocate a debug proxy through the allocator inside their - // noexcept move constructors, so failing that allocation terminates the - // program instead of throwing std::bad_alloc. Nothing to check there. + // noexcept default and move constructors, so failing that allocation + // terminates the program instead of throwing std::bad_alloc. Nothing to + // check there. #if !(defined(_ITERATOR_DEBUG_LEVEL) && _ITERATOR_DEBUG_LEVEL > 0) + // Clang 3.4 and 3.5 leak memory when an element constructor throws + // inside libstdc++'s containers (a std::map copy, a std::vector + // reallocation), without any json value involved. +#if !(defined(__clang__) && !defined(__apple_build_version__) && __clang_major__ == 3 && __clang_minor__ < 6) SECTION("parser callback") { const auto keep_all = [](int /*depth*/, sweep_json::parse_event_t /*event*/, sweep_json& /*parsed*/) noexcept @@ -1178,7 +1191,6 @@ TEST_CASE("parse() survives a failing allocation at every point (#5985)") return sweep_json::parse(text, drop_d); }); } -#endif SECTION("sax_parse into an existing value") { @@ -1192,5 +1204,7 @@ TEST_CASE("parse() survives a failing allocation at every point (#5985)") return value; }); } +#endif +#endif } #endif diff --git a/tests/src/unit-class_lexer.cpp b/tests/src/unit-class_lexer.cpp index 0e8b22317..616999896 100644 --- a/tests/src/unit-class_lexer.cpp +++ b/tests/src/unit-class_lexer.cpp @@ -2222,9 +2222,10 @@ TEST_CASE("serialization of short strings") { CAPTURE(s) const json j = s; - CHECK(j.dump() == "\"" + s + "\""); - CHECK(j.dump(-1, ' ', true) == "\"" + s + "\""); - CHECK(json({{s, s}}).dump() == "{\"" + s + "\":\"" + s + "\"}"); + const std::string quoted = "\"" + s + "\""; // NOLINT(performance-inefficient-string-concatenation) + CHECK(j.dump() == quoted); + CHECK(j.dump(-1, ' ', true) == quoted); + CHECK(json({{s, s}}).dump() == "{" + quoted + ":" + quoted + "}"); // NOLINT(performance-inefficient-string-concatenation) } // a character that needs an escape diff --git a/tests/src/unit-custom-object-key-type.cpp b/tests/src/unit-custom-object-key-type.cpp index acb78d628..9149bbfb6 100644 --- a/tests/src/unit-custom-object-key-type.cpp +++ b/tests/src/unit-custom-object-key-type.cpp @@ -160,12 +160,12 @@ class strict_key } private: - std::string m_value{}; + std::string m_value; }; strict_key make_strict_key(const std::string& key) { - return strict_key(key.c_str()); + return {key.c_str()}; } template @@ -224,7 +224,9 @@ TEST_CASE("converting a value with keys that cannot be made from a std::string n } } - const strict_json converted = value; + // direct-initialization: a copy-initialization makes GCC 4.8 instantiate the + // copy constructor, which needs a range constructor strict_object lacks + const strict_json converted(value); // at() needs to make a std::string of a key for its error message const auto member = [](const strict_json & object, const char* name) -> const strict_json & diff --git a/tests/src/unit-json_view.cpp b/tests/src/unit-json_view.cpp index e383dbdac..65180ef55 100644 --- a/tests/src/unit-json_view.cpp +++ b/tests/src/unit-json_view.cpp @@ -105,19 +105,53 @@ template struct failing_allocator { using value_type = T; + // GCC 4.8's std::map takes these from the allocator instead of from + // std::allocator_traits + using pointer = T*; + using const_pointer = const T*; + using reference = T&; + using const_reference = const T&; + using size_type = std::size_t; + using difference_type = std::ptrdiff_t; + + template + struct rebind + { + using other = failing_allocator; + }; failing_allocator() = default; template explicit failing_allocator(const failing_allocator& /*other*/) noexcept {} - T* allocate(std::size_t /*n*/) // NOLINT(readability-convert-member-functions-to-static) + [[noreturn]] T* allocate(std::size_t /*n*/) // NOLINT(readability-convert-member-functions-to-static) { throw std::bad_alloc(); } void deallocate(T* /*p*/, std::size_t /*n*/) noexcept {} // NOLINT(readability-convert-member-functions-to-static) + // GCC 4.8's std::map calls these on the allocator directly instead of + // going through std::allocator_traits + std::size_t max_size() const noexcept // NOLINT(readability-convert-member-functions-to-static) + { + return (std::numeric_limits::max)() / sizeof(T); + } + + template + void construct(U* p, Args&& ... args) // NOLINT(readability-convert-member-functions-to-static) + { + ::new (static_cast(p)) U(std::forward(args)...); + } + + template + void destroy(U* p) noexcept // NOLINT(readability-convert-member-functions-to-static) + { + static_cast(p); // unused for trivially destructible types + p->~U(); + } + template bool operator==(const failing_allocator& /*other*/) const noexcept { @@ -270,7 +304,7 @@ TEST_CASE("json_view") "\"", "\"abc", "\"\\x\"", "\"\\u12\"", "\"\\ud800\"", "\"\\udc00\"", "\"\x01\"", "\"\xff\"", "\"\xc3\"", "[1]x", "/", "/*", "[\n 1,\n x\n]", // NOLINT(modernize-raw-string-literal) "1e400", "-1e400", "[1.7976931348623159e308]", "{\"a\":\n{\"b\": [1, 2,\n 3 x]}}", // (an error after every kind of value: the error is found by replaying the parser) - "[null, true, -1, 1, 1.5, \"s\", {\"a\": 1} 2]", "{\"a\": null, \"b\": -1, \"c\": 1.5, \"d\": \"s\", \"e\": {}, 1}" + R"([null, true, -1, 1, 1.5, "s", {"a": 1} 2])", R"({"a": null, "b": -1, "c": 1.5, "d": "s", "e": {}, 1})" }) { CAPTURE(text) @@ -1556,7 +1590,7 @@ TEST_CASE("json_view values") CHECK_THROWS_WITH_AS(not_an_object.root().get(), json_view_test::message("type_error.302", "", 0, 5, "type must be object, but is array").c_str(), json::type_error&); #if !defined(JSON_NOEXCEPTION) // (and one that cannot allocate its nodes) - using failing_map = std::map, failing_allocator>>; + using failing_map = std::map, failing_allocator>>; // NOLINT(modernize-use-transparent-functors) const json_document one_member = json_document::parse(R"({"a":1})"); CHECK_THROWS_AS(one_member.root().get(), std::bad_alloc&); #endif diff --git a/tests/src/unit-locale-cpp.cpp b/tests/src/unit-locale-cpp.cpp index 19c08166c..d07eabeff 100644 --- a/tests/src/unit-locale-cpp.cpp +++ b/tests/src/unit-locale-cpp.cpp @@ -525,7 +525,7 @@ struct locale_switching_token { switched = std::setlocale(LC_NUMERIC, switch_to.c_str()) != nullptr; } - return &text[0]; + return &text[0]; // NOLINT(readability-container-data-pointer): std::string::data() is const before C++17 } std::string text; @@ -647,7 +647,7 @@ TEST_CASE("convert_float_locale_aware") // "C" -> comma locale: strtod stops at the '.', so the conversion is repeated { - locale_switching_token token("1.5", comma_locale.c_str()); + locale_switching_token token("1.5", comma_locale); double value = 0; nlohmann::detail::convert_float_locale_aware(token, 1, value); if (token.switched) @@ -660,7 +660,7 @@ TEST_CASE("convert_float_locale_aware") // the same for the other floating-point types REQUIRE(std::setlocale(LC_NUMERIC, "C") != nullptr); { - locale_switching_token token("2.25", comma_locale.c_str()); + locale_switching_token token("2.25", comma_locale); float value = 0; nlohmann::detail::convert_float_locale_aware(token, 1, value); if (token.switched) @@ -670,7 +670,7 @@ TEST_CASE("convert_float_locale_aware") } REQUIRE(std::setlocale(LC_NUMERIC, "C") != nullptr); { - locale_switching_token token("2.25", comma_locale.c_str()); + locale_switching_token token("2.25", comma_locale); long double value = 0; nlohmann::detail::convert_float_locale_aware(token, 1, value); if (token.switched) diff --git a/tests/src/unit-msgpack-cpp17.cpp b/tests/src/unit-msgpack-cpp17.cpp index 6b6b207bf..8b2349b90 100644 --- a/tests/src/unit-msgpack-cpp17.cpp +++ b/tests/src/unit-msgpack-cpp17.cpp @@ -122,7 +122,8 @@ TEST_CASE("JSON text with std::byte") SECTION("invalid text") { CHECK_FALSE(json::accept(to_bytes("[1,\n2,"))); - CHECK_THROWS_WITH_AS(json::parse(to_bytes("[1,\n2,")), + json _; + CHECK_THROWS_WITH_AS(_ = json::parse(to_bytes("[1,\n2,")), "[json.exception.parse_error.101] parse error at line 2, column 3: syntax error while parsing value - unexpected end of input; expected '[', '{', or a literal", json::parse_error&); } } diff --git a/tests/src/unit-ordered_json.cpp b/tests/src/unit-ordered_json.cpp index 8c07ffa16..4d894e649 100644 --- a/tests/src/unit-ordered_json.cpp +++ b/tests/src/unit-ordered_json.cpp @@ -541,6 +541,7 @@ TEST_CASE("object_key_index") }; std::vector hashes; + hashes.reserve(initial); for (std::size_t i = 0; i < initial; ++i) { hashes.push_back(hash_of(i)); diff --git a/tests/src/unit-regression1.cpp b/tests/src/unit-regression1.cpp index 53f501f07..367f52dfa 100644 --- a/tests/src/unit-regression1.cpp +++ b/tests/src/unit-regression1.cpp @@ -1533,7 +1533,7 @@ TEST_CASE("regression tests 1") { my_json::object_t top = orig.get_ref(); top.at("x") = my_json::parse(R"({"a":{"b":1},"n":null,"arr":[10,20],"objs":[{"m":5},{"m":6}]})"); - return my_json(std::move(top)); + return my_json(std::move(top)); // NOLINT(modernize-return-braced-init-list): {top} would be an array }; SECTION("existing members") diff --git a/tests/src/unit-stream-bulk-scan.cpp b/tests/src/unit-stream-bulk-scan.cpp index 0e0fbd72e..f6869847d 100644 --- a/tests/src/unit-stream-bulk-scan.cpp +++ b/tests/src/unit-stream-bulk-scan.cpp @@ -120,6 +120,7 @@ class oversized_get_area_streambuf : public std::streambuf private: std::array m_data{{'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'}}; }; + #endif // how a test reads from the stream @@ -463,15 +464,18 @@ TEST_CASE("parsing from a std::istream scans the get area in bulk") CHECK(token.empty()); } +#if SIZE_MAX > UINT32_MAX + // a get area larger than an int needs a 64-bit std::size_t (checked by the + // preprocessor, as a constant condition triggers MSVC's C4127) SECTION("skipping more characters than gbump() takes at once") { // gbump() takes an int, but a get area can be larger - const std::size_t claimed = sizeof(std::size_t) < 8 ? 0 : std::size_t{3} << 30u; + const std::size_t claimed = std::size_t{3} << 30u; oversized_get_area_streambuf buf(claimed); - if (claimed == 0 || static_cast(buf.in_avail()) != claimed) + if (static_cast(buf.in_avail()) != claimed) { - // a 32-bit size_t, or a standard library whose streambuf stores - // the size of the get area in an int (MSVC) + // a standard library whose streambuf stores the size of the get + // area in an int (MSVC) MESSAGE("a get area larger than an int is not supported"); } else @@ -491,6 +495,7 @@ TEST_CASE("parsing from a std::istream scans the get area in bulk") buf.restore_get_area(); } } +#endif #endif SECTION("SAX parsing from a stream") From 92707f43f2b1a2dffd26b676204c2237f2be2e42 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Nov 2026 19:17:27 +0100 Subject: [PATCH 42/54] build(deps): bump step-security/harden-runner from 2.22.2 to 2.22.3 (#6021) --- .github/workflows/cancel_closed_pr_runs.yml | 2 +- .github/workflows/check_amalgamation.yml | 4 ++-- .github/workflows/check_docs_links.yml | 2 +- .github/workflows/cifuzz.yml | 2 +- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/comment_check_amalgamation.yml | 2 +- .github/workflows/dependency-review.yml | 2 +- .github/workflows/flawfinder.yml | 2 +- .github/workflows/json_view_benchmarks.yml | 2 +- .github/workflows/labeler.yml | 2 +- .github/workflows/publish_documentation.yml | 2 +- .github/workflows/scorecards.yml | 2 +- .github/workflows/semgrep.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/ubuntu.yml | 10 +++++----- 15 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/cancel_closed_pr_runs.yml b/.github/workflows/cancel_closed_pr_runs.yml index 91005de52..fd30c5047 100644 --- a/.github/workflows/cancel_closed_pr_runs.yml +++ b/.github/workflows/cancel_closed_pr_runs.yml @@ -28,7 +28,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit diff --git a/.github/workflows/check_amalgamation.yml b/.github/workflows/check_amalgamation.yml index 253531e40..4fbec781d 100644 --- a/.github/workflows/check_amalgamation.yml +++ b/.github/workflows/check_amalgamation.yml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-26.04 steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit @@ -46,7 +46,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit diff --git a/.github/workflows/check_docs_links.yml b/.github/workflows/check_docs_links.yml index f7e28ab69..0ec9778e6 100644 --- a/.github/workflows/check_docs_links.yml +++ b/.github/workflows/check_docs_links.yml @@ -20,7 +20,7 @@ jobs: timeout-minutes: 30 steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit diff --git a/.github/workflows/cifuzz.yml b/.github/workflows/cifuzz.yml index f5feb9a8f..02a5a8173 100644 --- a/.github/workflows/cifuzz.yml +++ b/.github/workflows/cifuzz.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 61036d79d..99541a96f 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -27,7 +27,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit diff --git a/.github/workflows/comment_check_amalgamation.yml b/.github/workflows/comment_check_amalgamation.yml index ba2fcad7b..558d5fba4 100644 --- a/.github/workflows/comment_check_amalgamation.yml +++ b/.github/workflows/comment_check_amalgamation.yml @@ -20,7 +20,7 @@ jobs: pull-requests: write steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index 27c229572..8034aa1c8 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -21,7 +21,7 @@ jobs: runs-on: ubuntu-26.04 steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit diff --git a/.github/workflows/flawfinder.yml b/.github/workflows/flawfinder.yml index ffc8c2a43..2fae15234 100644 --- a/.github/workflows/flawfinder.yml +++ b/.github/workflows/flawfinder.yml @@ -31,7 +31,7 @@ jobs: security-events: write steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit diff --git a/.github/workflows/json_view_benchmarks.yml b/.github/workflows/json_view_benchmarks.yml index d87357c42..49876a07b 100644 --- a/.github/workflows/json_view_benchmarks.yml +++ b/.github/workflows/json_view_benchmarks.yml @@ -54,7 +54,7 @@ jobs: timeout-minutes: 60 steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 85c7a513d..004039586 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -23,7 +23,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit diff --git a/.github/workflows/publish_documentation.yml b/.github/workflows/publish_documentation.yml index ad53308d7..ba8f435bf 100644 --- a/.github/workflows/publish_documentation.yml +++ b/.github/workflows/publish_documentation.yml @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-26.04 steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index 62e1d504a..47cde1be7 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -40,7 +40,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml index 8b1e6ed5e..ce5c115b4 100644 --- a/.github/workflows/semgrep.yml +++ b/.github/workflows/semgrep.yml @@ -36,7 +36,7 @@ jobs: runs-on: ubuntu-26.04 steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index da7d91fce..b624f9f40 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index c93c0a706..5dd479bcd 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -80,7 +80,7 @@ jobs: runs-on: ubuntu-26.04 steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit @@ -112,7 +112,7 @@ jobs: target: [ci_test_single_header, ci_cppcheck, ci_cpplint, ci_reproducible_tests, ci_non_git_tests, ci_offline_testdata, ci_reuse_compliance, ci_test_valgrind] steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit @@ -178,7 +178,7 @@ jobs: runs-on: ubuntu-26.04 steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit @@ -433,7 +433,7 @@ jobs: runs-on: ubuntu-26.04 steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit @@ -456,7 +456,7 @@ jobs: target: [ci_test_examples, ci_test_build_documentation, ci_test_documentation_mermaid] steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit From 57b898ef9989ea99afc090bc96e435c845993e61 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Tue, 3 Nov 2026 19:17:41 +0100 Subject: [PATCH 43/54] Replace a multibyte decimal point in dump() of a long double (#5998) --- include/nlohmann/detail/output/serializer.hpp | 42 ++++++++++--------- single_include/nlohmann/json.hpp | 42 ++++++++++--------- tests/src/unit-locale-cpp.cpp | 8 ++++ 3 files changed, 54 insertions(+), 38 deletions(-) diff --git a/include/nlohmann/detail/output/serializer.hpp b/include/nlohmann/detail/output/serializer.hpp index 0c004c491..235e7bafd 100644 --- a/include/nlohmann/detail/output/serializer.hpp +++ b/include/nlohmann/detail/output/serializer.hpp @@ -8,7 +8,7 @@ #pragma once -#include // remove, fill, find, none_of, min +#include // copy, none_of, min, search #include // array #include // localeconv, lconv #include // isfinite @@ -2080,31 +2080,35 @@ class serializer // check if the buffer was large enough JSON_ASSERT(static_cast(len) < number_buffer.size()); - // look up the locale's thousands separator and decimal point now, - // matching what snprintf_float() just used (see lexer::get_decimal_point()) + // look up the locale's decimal point now, matching what + // snprintf_float() just used (see get_decimal_point() in + // number_parse.hpp); it is a string, which can be longer than one + // byte (U+066B in ps_AF, #5993). + // A thousands separator needs no handling: %g never groups digits. const auto* loc = std::localeconv(); JSON_ASSERT(loc != nullptr); - const char thousands_sep = (loc->thousands_sep == nullptr) ? '\0' : *loc->thousands_sep; - const char decimal_point = (loc->decimal_point == nullptr) ? '\0' : *loc->decimal_point; - - // erase thousands separators - if (thousands_sep != '\0') - { - // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::remove returns an iterator, see https://github.com/nlohmann/json/issues/3081 - const auto end = std::remove(number_buffer.begin(), number_buffer.begin() + len, thousands_sep); - std::fill(end, number_buffer.end(), '\0'); - JSON_ASSERT((end - number_buffer.begin()) <= len); - len = (end - number_buffer.begin()); - } + const char* const decimal_point = (loc->decimal_point == nullptr) ? "" : loc->decimal_point; + const std::size_t decimal_point_len = std::char_traits::length(decimal_point); // convert decimal point to '.' - if (decimal_point != '\0' && decimal_point != '.') + if ((decimal_point_len > 1) || (decimal_point_len == 1 && decimal_point[0] != '.')) { - // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::find returns an iterator, see https://github.com/nlohmann/json/issues/3081 - const auto dec_pos = std::find(number_buffer.begin(), number_buffer.end(), decimal_point); - if (dec_pos != number_buffer.end()) + const auto first = number_buffer.begin(); + const auto last = first + len; + // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::search returns an iterator, see https://github.com/nlohmann/json/issues/3081 + const auto dec_pos = std::search(first, last, decimal_point, decimal_point + decimal_point_len); + if (dec_pos != last) { *dec_pos = '.'; + if (decimal_point_len > 1) + { + // move the fraction and exponent behind the '.' (std::copy + // allows the overlap, as the destination is before the source) + // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::copy returns an iterator, see https://github.com/nlohmann/json/issues/3081 + const auto end = std::copy(dec_pos + static_cast(decimal_point_len), last, dec_pos + 1); + *end = '\0'; + len = end - first; + } } } diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index cf5a4d711..952cea7b6 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -33446,7 +33446,7 @@ NLOHMANN_JSON_NAMESPACE_END -#include // remove, fill, find, none_of, min +#include // copy, none_of, min, search #include // array #include // localeconv, lconv #include // isfinite @@ -37300,31 +37300,35 @@ class serializer // check if the buffer was large enough JSON_ASSERT(static_cast(len) < number_buffer.size()); - // look up the locale's thousands separator and decimal point now, - // matching what snprintf_float() just used (see lexer::get_decimal_point()) + // look up the locale's decimal point now, matching what + // snprintf_float() just used (see get_decimal_point() in + // number_parse.hpp); it is a string, which can be longer than one + // byte (U+066B in ps_AF, #5993). + // A thousands separator needs no handling: %g never groups digits. const auto* loc = std::localeconv(); JSON_ASSERT(loc != nullptr); - const char thousands_sep = (loc->thousands_sep == nullptr) ? '\0' : *loc->thousands_sep; - const char decimal_point = (loc->decimal_point == nullptr) ? '\0' : *loc->decimal_point; - - // erase thousands separators - if (thousands_sep != '\0') - { - // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::remove returns an iterator, see https://github.com/nlohmann/json/issues/3081 - const auto end = std::remove(number_buffer.begin(), number_buffer.begin() + len, thousands_sep); - std::fill(end, number_buffer.end(), '\0'); - JSON_ASSERT((end - number_buffer.begin()) <= len); - len = (end - number_buffer.begin()); - } + const char* const decimal_point = (loc->decimal_point == nullptr) ? "" : loc->decimal_point; + const std::size_t decimal_point_len = std::char_traits::length(decimal_point); // convert decimal point to '.' - if (decimal_point != '\0' && decimal_point != '.') + if ((decimal_point_len > 1) || (decimal_point_len == 1 && decimal_point[0] != '.')) { - // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::find returns an iterator, see https://github.com/nlohmann/json/issues/3081 - const auto dec_pos = std::find(number_buffer.begin(), number_buffer.end(), decimal_point); - if (dec_pos != number_buffer.end()) + const auto first = number_buffer.begin(); + const auto last = first + len; + // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::search returns an iterator, see https://github.com/nlohmann/json/issues/3081 + const auto dec_pos = std::search(first, last, decimal_point, decimal_point + decimal_point_len); + if (dec_pos != last) { *dec_pos = '.'; + if (decimal_point_len > 1) + { + // move the fraction and exponent behind the '.' (std::copy + // allows the overlap, as the destination is before the source) + // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::copy returns an iterator, see https://github.com/nlohmann/json/issues/3081 + const auto end = std::copy(dec_pos + static_cast(decimal_point_len), last, dec_pos + 1); + *end = '\0'; + len = end - first; + } } } diff --git a/tests/src/unit-locale-cpp.cpp b/tests/src/unit-locale-cpp.cpp index d07eabeff..10143326a 100644 --- a/tests/src/unit-locale-cpp.cpp +++ b/tests/src/unit-locale-cpp.cpp @@ -397,6 +397,14 @@ TEST_CASE("locale with a multi-byte decimal point") // a value the locale-independent paths convert is not affected CHECK(json::parse("12.5") == 12.5); + + // dump() of a long double that takes the snprintf path replaces the + // whole decimal point, not only its first byte (#5993) + CHECK(long_double_json(1.5L).dump() == "1.5"); + CHECK(long_double_json(2.0L).dump() == "2.0"); + CHECK(long_double_json(-0.25L).dump() == "-0.25"); + CHECK(long_double_json::parse(long_double_json(1.5e-7L).dump()) == long_double_json(1.5e-7L)); + CHECK(long_double_json::parse(ld.dump()) == expected_long_double); } if (!tested) { From efc437ef6e25d0e5617c4b6385a095cb383a5a2b Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Wed, 4 Nov 2026 17:13:16 +0100 Subject: [PATCH 44/54] UBJSON/BJData/MessagePack reader: reject a container count equal to SIZE_MAX (#6002) * UBJSON/BJData reader: reject a container count equal to SIZE_MAX The reader marks a container whose count is not known by npos, which is SIZE_MAX. A count of exactly that value was stored unchanged and then read as "no count", so the BJData input [ # M ff ff ff ff ff ff ff ff i 01 ] parsed as [1] instead of throwing out_of_range.408 as a count of 2^64-2 does. Where size_t is 32 bits, the same holds for 'm' and 'L' counts of 2^32-1, in UBJSON too. The counts of 'm', 'M' and the signed types now go through store_ubjson_count(), which rejects npos with "integer value overflow" (the message a count beyond size_t already gets), as the CBOR reader does in get_cbor_container_size(). An ND-array dimension of 2^64-1 gets the same message now, where it was "excessive ndarray size caused overflow" (the product check) on 64-bit platforms. Signed-off-by: Niels Lohmann * MessagePack reader: reject an array/map 32 length equal to SIZE_MAX On a platform with a 32-bit std::size_t, an array 32 or map 32 length of 0xFFFFFFFF equals unknown_size(), so start_array()/start_object() were told the size was unknown and skipped the max_size() check. Route both through the CBOR length check, renamed get_container_size(), which already rejects that value. Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann --- .../nlohmann/detail/input/binary_reader.hpp | 44 ++++++++++++++----- single_include/nlohmann/json.hpp | 44 ++++++++++++++----- tests/src/unit-32bit.cpp | 20 +++++++++ tests/src/unit-bjdata.cpp | 28 ++++++++++-- tests/src/unit-regression2.cpp | 2 +- tests/src/unit-ubjson.cpp | 12 +++++ 6 files changed, 121 insertions(+), 29 deletions(-) diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index 8edc53ae9..b793ae097 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -1024,7 +1024,7 @@ class binary_reader { std::uint64_t len{}; std::size_t size{}; - return get_cbor_argument(len) && get_cbor_container_size(len, size, "array") && enter_cbor_array(size); + return get_cbor_argument(len) && get_container_size(len, size, "array") && enter_cbor_array(size); } case 0x9F: // array (indefinite length) @@ -1064,7 +1064,7 @@ class binary_reader { std::uint64_t len{}; std::size_t size{}; - return get_cbor_argument(len) && get_cbor_container_size(len, size, "map") && enter_object(size); + return get_cbor_argument(len) && get_container_size(len, size, "map") && enter_object(size); } case 0xBF: // map (indefinite length) @@ -1700,7 +1700,7 @@ class binary_reader } /*! - @brief narrow a definite CBOR array/map length to std::size_t + @brief narrow a definite CBOR or MessagePack array/map length to std::size_t A definite length is rejected if it does not fit in std::size_t or if it equals detail::unknown_size(), which is reserved to mark an indefinite- @@ -1713,7 +1713,7 @@ class binary_reader @param[in] context "array" or "map", for the error message @return whether the length is usable */ - bool get_cbor_container_size(const std::uint64_t len, std::size_t& result, const char* context) + bool get_container_size(const std::uint64_t len, std::size_t& result, const char* context) { if (JSON_HEDLEY_UNLIKELY(!value_in_range_of(len) || len == detail::unknown_size())) { @@ -2309,7 +2309,8 @@ class binary_reader case 0xDD: // array 32 { std::uint32_t len{}; - return get_number(len) && enter_msgpack_array(conditional_static_cast(len)); + std::size_t size{}; + return get_number(len) && get_container_size(len, size, "array") && enter_msgpack_array(size); } case 0xDE: // map 16 @@ -2321,7 +2322,8 @@ class binary_reader case 0xDF: // map 32 { std::uint32_t len{}; - return get_number(len) && enter_object(conditional_static_cast(len)); + std::size_t size{}; + return get_number(len) && get_container_size(len, size, "map") && enter_object(size); } // negative fixint @@ -3131,6 +3133,27 @@ class binary_reader return true; } + /*! + @brief store the count of an optimized container (or the length of a string) + + A count equal to npos is rejected: npos marks a container whose count is + not known, so a count of that value would be read as "no count" (compare + get_container_size()). No container can hold that many elements. + + @param[in] count the count read from the input + @param[out] result set to @a count unless it is rejected + @return whether @a count is usable + */ + bool store_ubjson_count(const std::size_t count, std::size_t& result) + { + if (JSON_HEDLEY_UNLIKELY(count == npos)) + { + return out_of_range_error(exception_id::container_too_large, "integer value overflow", "size"); + } + result = count; + return true; + } + /*! @brief read a UBJSON/BJData optimized-container count of a signed marker type ('i', 'I', 'l', 'L') and narrow it to std::size_t @@ -3161,8 +3184,7 @@ class binary_reader { return out_of_range_error(exception_id::container_too_large, "integer value overflow", "size"); } - result = static_cast(number); // NOLINT(bugprone-signed-char-misuse,cert-str34-c): number is not a char - return true; + return store_ubjson_count(static_cast(number), result); // NOLINT(bugprone-signed-char-misuse,cert-str34-c): number is not a char } /*! @@ -3239,8 +3261,7 @@ class binary_reader { return false; } - result = conditional_static_cast(number); - return true; + return store_ubjson_count(conditional_static_cast(number), result); } case 'M': @@ -3258,8 +3279,7 @@ class binary_reader { return out_of_range_error(exception_id::container_too_large, "integer value overflow", "size"); } - result = detail::conditional_static_cast(number); - return true; + return store_ubjson_count(detail::conditional_static_cast(number), result); } case '[': diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 952cea7b6..48c99397b 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -19039,7 +19039,7 @@ class binary_reader { std::uint64_t len{}; std::size_t size{}; - return get_cbor_argument(len) && get_cbor_container_size(len, size, "array") && enter_cbor_array(size); + return get_cbor_argument(len) && get_container_size(len, size, "array") && enter_cbor_array(size); } case 0x9F: // array (indefinite length) @@ -19079,7 +19079,7 @@ class binary_reader { std::uint64_t len{}; std::size_t size{}; - return get_cbor_argument(len) && get_cbor_container_size(len, size, "map") && enter_object(size); + return get_cbor_argument(len) && get_container_size(len, size, "map") && enter_object(size); } case 0xBF: // map (indefinite length) @@ -19715,7 +19715,7 @@ class binary_reader } /*! - @brief narrow a definite CBOR array/map length to std::size_t + @brief narrow a definite CBOR or MessagePack array/map length to std::size_t A definite length is rejected if it does not fit in std::size_t or if it equals detail::unknown_size(), which is reserved to mark an indefinite- @@ -19728,7 +19728,7 @@ class binary_reader @param[in] context "array" or "map", for the error message @return whether the length is usable */ - bool get_cbor_container_size(const std::uint64_t len, std::size_t& result, const char* context) + bool get_container_size(const std::uint64_t len, std::size_t& result, const char* context) { if (JSON_HEDLEY_UNLIKELY(!value_in_range_of(len) || len == detail::unknown_size())) { @@ -20324,7 +20324,8 @@ class binary_reader case 0xDD: // array 32 { std::uint32_t len{}; - return get_number(len) && enter_msgpack_array(conditional_static_cast(len)); + std::size_t size{}; + return get_number(len) && get_container_size(len, size, "array") && enter_msgpack_array(size); } case 0xDE: // map 16 @@ -20336,7 +20337,8 @@ class binary_reader case 0xDF: // map 32 { std::uint32_t len{}; - return get_number(len) && enter_object(conditional_static_cast(len)); + std::size_t size{}; + return get_number(len) && get_container_size(len, size, "map") && enter_object(size); } // negative fixint @@ -21146,6 +21148,27 @@ class binary_reader return true; } + /*! + @brief store the count of an optimized container (or the length of a string) + + A count equal to npos is rejected: npos marks a container whose count is + not known, so a count of that value would be read as "no count" (compare + get_container_size()). No container can hold that many elements. + + @param[in] count the count read from the input + @param[out] result set to @a count unless it is rejected + @return whether @a count is usable + */ + bool store_ubjson_count(const std::size_t count, std::size_t& result) + { + if (JSON_HEDLEY_UNLIKELY(count == npos)) + { + return out_of_range_error(exception_id::container_too_large, "integer value overflow", "size"); + } + result = count; + return true; + } + /*! @brief read a UBJSON/BJData optimized-container count of a signed marker type ('i', 'I', 'l', 'L') and narrow it to std::size_t @@ -21176,8 +21199,7 @@ class binary_reader { return out_of_range_error(exception_id::container_too_large, "integer value overflow", "size"); } - result = static_cast(number); // NOLINT(bugprone-signed-char-misuse,cert-str34-c): number is not a char - return true; + return store_ubjson_count(static_cast(number), result); // NOLINT(bugprone-signed-char-misuse,cert-str34-c): number is not a char } /*! @@ -21254,8 +21276,7 @@ class binary_reader { return false; } - result = conditional_static_cast(number); - return true; + return store_ubjson_count(conditional_static_cast(number), result); } case 'M': @@ -21273,8 +21294,7 @@ class binary_reader { return out_of_range_error(exception_id::container_too_large, "integer value overflow", "size"); } - result = detail::conditional_static_cast(number); - return true; + return store_ubjson_count(detail::conditional_static_cast(number), result); } case '[': diff --git a/tests/src/unit-32bit.cpp b/tests/src/unit-32bit.cpp index 97549c999..e4e2dea6d 100644 --- a/tests/src/unit-32bit.cpp +++ b/tests/src/unit-32bit.cpp @@ -97,3 +97,23 @@ TEST_CASE("CBOR") } } } + +TEST_CASE("MessagePack") +{ + SECTION("parse errors") + { + SECTION("array/map size equal to the unknown-size sentinel") + { + // on 32-bit platforms an array 32/map 32 length of 0xFFFFFFFF aliases unknown_size() + std::vector const varr = {0xDD, 0xFF, 0xFF, 0xFF, 0xFF}; + std::vector const vmap = {0xDF, 0xFF, 0xFF, 0xFF, 0xFF}; + + json _; + CHECK_THROWS_WITH_AS(_ = json::from_msgpack(varr), "[json.exception.out_of_range.408] syntax error while parsing MessagePack size: excessive array size", json::out_of_range&); + CHECK(json::from_msgpack(varr, true, false).is_discarded()); + + CHECK_THROWS_WITH_AS(_ = json::from_msgpack(vmap), "[json.exception.out_of_range.408] syntax error while parsing MessagePack size: excessive map size", json::out_of_range&); + CHECK(json::from_msgpack(vmap, true, false).is_discarded()); + } + } +} diff --git a/tests/src/unit-bjdata.cpp b/tests/src/unit-bjdata.cpp index ec7faa6cd..ecc79959d 100644 --- a/tests/src/unit-bjdata.cpp +++ b/tests/src/unit-bjdata.cpp @@ -3237,14 +3237,34 @@ TEST_CASE("BJData") #endif CHECK(json::from_bjdata(vM, true, false).is_discarded()); -#if SIZE_MAX != 0xffffffff - CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vMX), "[json.exception.out_of_range.408] syntax error while parsing BJData size: excessive ndarray size caused overflow", json::out_of_range&); -#else + // the first dimension is 2^64-1, which is rejected as a count (see below) CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vMX), "[json.exception.out_of_range.408] syntax error while parsing BJData size: integer value overflow", json::out_of_range&); -#endif CHECK(json::from_bjdata(vMX, true, false).is_discarded()); } + SECTION("optimized array: count equal to SIZE_MAX") + { + // the reader marks a container without a count by SIZE_MAX, so + // a count of that value must not be read as "no count" + // (it used to parse as [1]) + json _; +#if SIZE_MAX != 0xffffffff + std::vector const vM = {'[', '#', 'M', 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 'i', 0x01, ']'}; + CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vM), "[json.exception.out_of_range.408] syntax error while parsing BJData size: integer value overflow", json::out_of_range&); + CHECK(json::from_bjdata(vM, true, false).is_discarded()); + // one less is an ordinary excessive count + std::vector const vM1 = {'[', '#', 'M', 0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 'i', 0x01, ']'}; + CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vM1), "[json.exception.out_of_range.408] excessive array size: 18446744073709551614", json::out_of_range&); +#else + std::vector const vm = {'[', '#', 'm', 0xFF, 0xFF, 0xFF, 0xFF, 'i', 0x01, ']'}; + CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vm), "[json.exception.out_of_range.408] syntax error while parsing BJData size: integer value overflow", json::out_of_range&); + CHECK(json::from_bjdata(vm, true, false).is_discarded()); + std::vector const vL = {'[', '#', 'L', 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 'i', 0x01, ']'}; + CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vL), "[json.exception.out_of_range.408] syntax error while parsing BJData size: integer value overflow", json::out_of_range&); + CHECK(json::from_bjdata(vL, true, false).is_discarded()); +#endif + } + SECTION("optimized array: integer value overflow") { #if SIZE_MAX == 0xffffffff diff --git a/tests/src/unit-regression2.cpp b/tests/src/unit-regression2.cpp index 08d342d0a..f7bd557e8 100644 --- a/tests/src/unit-regression2.cpp +++ b/tests/src/unit-regression2.cpp @@ -1009,7 +1009,7 @@ TEST_CASE("regression test - excessive binary container size honors allow_except // allow_exceptions=true (the default) must still throw exactly as before. // The exact message text is not checked here: on platforms where // std::size_t is 32-bit, the CBOR reader's own length-narrowing check - // (get_cbor_container_size(), unrelated to this fix) intercepts a + // (get_container_size(), unrelated to this fix) intercepts a // declared length of 2^63 before it ever reaches the check this test // targets, with different (but equally valid, and already correct) // wording -- see unit-cbor.cpp for coverage of that message. diff --git a/tests/src/unit-ubjson.cpp b/tests/src/unit-ubjson.cpp index 4bc7f7ac0..b4fdc0997 100644 --- a/tests/src/unit-ubjson.cpp +++ b/tests/src/unit-ubjson.cpp @@ -2385,6 +2385,18 @@ TEST_CASE("UBJSON optimized arrays of a valueless type are bounded") } } + SECTION("a count equal to SIZE_MAX is rejected") + { + // the reader marks a container without a count by SIZE_MAX, so a + // count of that value must not be read as "no count"; UBJSON's + // largest count type (int64) can only reach it where size_t is 32 bits +#if SIZE_MAX == 0xffffffff + const std::vector input = {'[', '#', 'L', 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, 'i', 0x01, ']'}; + CHECK_THROWS_WITH_AS(_ = json::from_ubjson(input), "[json.exception.out_of_range.408] syntax error while parsing UBJSON size: integer value overflow", json::out_of_range&); + CHECK(json::from_ubjson(input, true, false).is_discarded()); +#endif + } + SECTION("ordinary counts are unaffected") { CHECK(json::from_ubjson(std::vector({'[', '$', 'Z', '#', 'i', 3})) == json({nullptr, nullptr, nullptr})); From 42825fba0744d0169c59a18c4bca6f8e3155be20 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Wed, 4 Nov 2026 17:13:25 +0100 Subject: [PATCH 45/54] Binary writers: zero only a window ahead of the vector sink's position (#6020) output_vector_sink grew the vector by resizing it to its whole new capacity. resize() zeroes the characters it adds, so every doubling zeroed megabytes of a fresh allocation at once: that faults in all its pages and evicts them from the cache before the writer reaches them, and the writer's stores then miss again. The sink now reserves the same capacity as before but resizes the vector only to 64 KiB (vector_sink_window) ahead of the position, which is still cached when the writer's stores come. to_cbor(), to_msgpack(), to_ubjson(), to_bjdata() and to_bon8() of an array of one million numbers into a fresh vector: 4-26 % faster on x86 (Skylake laptop with Apple clang 13, Haswell VPS with GCC 16 and Clang 23), 1-2.5 % on Apple M1 Max; other documents unchanged (within 0.4 % in reruns). The capacity and growth of the vector are as before. Signed-off-by: Niels Lohmann --- .../detail/output/output_adapters.hpp | 21 +++++++++++++++---- single_include/nlohmann/json.hpp | 21 +++++++++++++++---- tests/src/unit-binary_writer_sinks.cpp | 10 ++++++++- 3 files changed, 43 insertions(+), 9 deletions(-) diff --git a/include/nlohmann/detail/output/output_adapters.hpp b/include/nlohmann/detail/output/output_adapters.hpp index e9e57ab9a..b1ea305b9 100644 --- a/include/nlohmann/detail/output/output_adapters.hpp +++ b/include/nlohmann/detail/output/output_adapters.hpp @@ -116,6 +116,10 @@ inline void copy_short(CharType* dst, const CharType* src, const std::size_t len /// the number of characters an empty vector is grown to by output_vector_sink constexpr std::size_t min_vector_sink_capacity = 64; +/// the number of characters output_vector_sink makes writable ahead of its +/// position at a time: 64 KiB of the vector's capacity, not all of it +constexpr std::size_t vector_sink_window = 65536; + /// @brief non-virtual output sink writing into a std::vector /// /// This sink is not part of the virtual output_adapter_protocol hierarchy: it is @@ -129,7 +133,8 @@ constexpr std::size_t min_vector_sink_capacity = 64; /// encoding them: insert() is an out-of-line call into a routine that can insert /// anywhere, and resize() is one as well with libstdc++. The sink therefore keeps /// its own write position: it grows the vector geometrically ahead of the -/// position and copies each write to it, so a write is a capacity check and a +/// position (making @ref vector_sink_window characters of it writable at a +/// time) and copies each write to it, so a write is a capacity check and a /// short copy. The destructor shrinks the vector to the bytes actually written, /// also when a write throws; shrinking a vector never allocates. The sink owns /// that trim, so it can be moved but not copied, and the vector must not be used @@ -277,9 +282,17 @@ class output_vector_sink // reallocate for every few bytes const std::size_t written = position(); const std::size_t needed = written + length; - v->resize((std::max)(min_vector_sink_capacity, needed <= v->capacity() - ? v->capacity() - : (std::max)(needed, v->capacity() * 2))); + const std::size_t capacity = (std::max)(min_vector_sink_capacity, needed <= v->capacity() + ? v->capacity() + : (std::max)(needed, v->capacity() * 2)); + v->reserve(capacity); + // resize() zeroes the characters it adds. Zeroing all of a doubled + // capacity, megabytes of a fresh allocation, at once faults in its + // pages and evicts them from the cache before the writer gets there, + // so its stores miss again; a window ahead of the position is still + // cached when they come (the binary writers on a long array of + // numbers: 4-26 % faster on x86, 1-2.5 % on Apple M1) + v->resize((std::min)(capacity, (std::max)(needed, written + vector_sink_window))); cursor = v->data() + written; limit = v->data() + v->size(); } diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 48c99397b..bb81909ae 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -27511,6 +27511,10 @@ inline void copy_short(CharType* dst, const CharType* src, const std::size_t len /// the number of characters an empty vector is grown to by output_vector_sink constexpr std::size_t min_vector_sink_capacity = 64; +/// the number of characters output_vector_sink makes writable ahead of its +/// position at a time: 64 KiB of the vector's capacity, not all of it +constexpr std::size_t vector_sink_window = 65536; + /// @brief non-virtual output sink writing into a std::vector /// /// This sink is not part of the virtual output_adapter_protocol hierarchy: it is @@ -27524,7 +27528,8 @@ constexpr std::size_t min_vector_sink_capacity = 64; /// encoding them: insert() is an out-of-line call into a routine that can insert /// anywhere, and resize() is one as well with libstdc++. The sink therefore keeps /// its own write position: it grows the vector geometrically ahead of the -/// position and copies each write to it, so a write is a capacity check and a +/// position (making @ref vector_sink_window characters of it writable at a +/// time) and copies each write to it, so a write is a capacity check and a /// short copy. The destructor shrinks the vector to the bytes actually written, /// also when a write throws; shrinking a vector never allocates. The sink owns /// that trim, so it can be moved but not copied, and the vector must not be used @@ -27672,9 +27677,17 @@ class output_vector_sink // reallocate for every few bytes const std::size_t written = position(); const std::size_t needed = written + length; - v->resize((std::max)(min_vector_sink_capacity, needed <= v->capacity() - ? v->capacity() - : (std::max)(needed, v->capacity() * 2))); + const std::size_t capacity = (std::max)(min_vector_sink_capacity, needed <= v->capacity() + ? v->capacity() + : (std::max)(needed, v->capacity() * 2)); + v->reserve(capacity); + // resize() zeroes the characters it adds. Zeroing all of a doubled + // capacity, megabytes of a fresh allocation, at once faults in its + // pages and evicts them from the cache before the writer gets there, + // so its stores miss again; a window ahead of the position is still + // cached when they come (the binary writers on a long array of + // numbers: 4-26 % faster on x86, 1-2.5 % on Apple M1) + v->resize((std::min)(capacity, (std::max)(needed, written + vector_sink_window))); cursor = v->data() + written; limit = v->data() + v->size(); } diff --git a/tests/src/unit-binary_writer_sinks.cpp b/tests/src/unit-binary_writer_sinks.cpp index 7ec2eb8fc..5638435ca 100644 --- a/tests/src/unit-binary_writer_sinks.cpp +++ b/tests/src/unit-binary_writer_sinks.cpp @@ -50,6 +50,14 @@ std::vector test_values() large_payloads.push_back(json::binary(std::vector(static_cast(100 * i), static_cast(i)))); } + // several times vector_sink_window characters in every format: the vector + // sink makes the capacity it grows to writable a window at a time + json many_windows = json::array(); + for (std::uint64_t i = 0; i < 30000; ++i) + { + many_windows.push_back((i * 0x9E3779B97F4A7C15u) >> 1u); + } + return { json(nullptr), json(true), json(false), @@ -61,7 +69,7 @@ std::vector test_values() json::array(), json::object(), json::array({1, 2, 3}), json({{"a", 1}, {"b", nullptr}}), json({{"nested", {{"deep", json::array({1, "two", 3.0, nullptr})}}}}), - big_array, big_object, large_payloads, + big_array, big_object, large_payloads, many_windows, json::binary(std::vector(5000, 0x42)) }; } From a49242f0880b5e1163ac89a2163c787ad6aca0eb Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Wed, 4 Nov 2026 17:46:20 +0100 Subject: [PATCH 46/54] Docs: reading and writing files; patch_inplace() is not atomic (#6017) --- .../docs/api/basic_json/patch_inplace.md | 19 ++- docs/mkdocs/docs/features/files.md | 130 ++++++++++++++++++ docs/mkdocs/docs/features/index.md | 2 + docs/mkdocs/docs/features/json_patch.md | 3 +- docs/mkdocs/docs/features/json_view.md | 4 +- docs/mkdocs/docs/features/parsing/index.md | 5 + docs/mkdocs/docs/features/performance.md | 4 +- docs/mkdocs/docs/features/serialization.md | 3 + docs/mkdocs/docs/home/faq.md | 2 +- docs/mkdocs/mkdocs.yml | 1 + 10 files changed, 166 insertions(+), 7 deletions(-) create mode 100644 docs/mkdocs/docs/features/files.md diff --git a/docs/mkdocs/docs/api/basic_json/patch_inplace.md b/docs/mkdocs/docs/api/basic_json/patch_inplace.md index ff65e8dbd..c3cd4f530 100644 --- a/docs/mkdocs/docs/api/basic_json/patch_inplace.md +++ b/docs/mkdocs/docs/api/basic_json/patch_inplace.md @@ -15,7 +15,9 @@ the patch. This function applies a JSON patch in place and returns void. ## Exception safety -No guarantees, value may be corrupted by an unsuccessful patch operation. +No guarantees: if an operation fails, the operations before it stay applied, and a failing `move` operation may already +have removed the value at its "from" location. Use [`patch`](patch.md) if the value must be unchanged after a failed +patch. ## Exceptions @@ -62,8 +64,19 @@ the JSON value, the complexity can usually be neglected. ## Notes Unlike [`patch`](patch.md), `patch_inplace` applies the operation "in place" and no copy of the JSON value is created. -That makes it faster for large documents by avoiding the copy. However, the JSON value might be corrupted if the -function throws an exception. +That makes it faster for large documents by avoiding the copy. + +!!! warning "Not atomic" + + `patch_inplace` applies the operations one after the other and does not undo them when one fails. After an + exception, the value holds the result of the operations before the failing one, and a failing `move` operation + (a "remove" followed by an "add") may already have removed the value at its "from" location; see the second example + below. [RFC 6902](https://tools.ietf.org/html/rfc6902#section-5) requires that a patch with a failing operation is + not deemed successful, but leaves open what happens to the document. + + If the value must stay unchanged when the patch fails, use [`patch`](patch.md), which applies the patch to a copy: + `#!cpp j = j.patch(json_patch);`. Validating the patch beforehand is not enough, as "test" operations and missing + paths are only detected while the patch is applied. Duplicate member names inside an operation object are resolved by the parser before the patch is applied (the last one is kept, see [`object_t`](object_t.md#behavior)), so `#!json {"op": "add", "op": "remove", "path": "/a"}` is a diff --git a/docs/mkdocs/docs/features/files.md b/docs/mkdocs/docs/features/files.md new file mode 100644 index 000000000..3833f13db --- /dev/null +++ b/docs/mkdocs/docs/features/files.md @@ -0,0 +1,130 @@ +# Reading and writing files + +The library has no functions that take a file name. It reads from and writes to streams, so files are handled with +`#!cpp std::ifstream` and `#!cpp std::ofstream` (or a `#!cpp FILE*` when reading). This page collects the recipes for +the common cases and the mistakes that show up again and again. + +## Reading a JSON file + +Pass the stream to [`parse`](../api/basic_json/parse.md) directly, and check that the file was opened: + +```cpp +#include +#include +#include +#include + +using json = nlohmann::json; + +json read_json_file(const std::string& path) +{ + std::ifstream f(path); + if (!f.is_open()) + { + throw std::runtime_error("cannot open " + path); + } + return json::parse(f); +} +``` + +- **Check that the file is open.** A stream that failed to open looks like an empty input, so `parse` reports + [`parse_error.101`](../home/exceptions.md#jsonexceptionparse_error101) ("unexpected end of input") at line 1, + column 1 instead of saying that the file is missing. A relative path is resolved against the working directory of the + running program, which is often not the directory of the source file (see the + [FAQ](../home/faq.md#parsing-a-file-fails-with-unexpected-end-of-input)). +- **Don't read the file into a string first.** Parsing from a `#!cpp std::ifstream` is only about 10 to 20 % slower + than reading the whole file into a string with one `read()` and parsing that, and it needs no memory for the copy (see + [input sources and performance](parsing/index.md#input-sources-and-performance)). The common way to read a file with + `#!cpp std::istreambuf_iterator` is so slow that it takes about twice as long as parsing from the stream. Read + the file into a string only if you need the text itself, for example for an error message. This advice is for + `#!cpp json::parse`. [`json_document`](json_view.md) reads a stream into a buffer that it owns anyway, so there is no + copy to save: reading the file into a `#!cpp std::string` yourself and passing that as an lvalue, which the document + borrows, was about 6 to 12 % faster than parsing from the stream, with the same single buffer. +- `parse` requires the file to hold exactly one JSON value. Files with one value per line are read with + [JSON Lines](parsing/json_lines.md); files too large for memory are covered in + [Large, streamed, and partial input](parsing/large_and_partial_input.md). + +## Writing a JSON file + +[`dump`](../api/basic_json/dump.md) the value into a string first, then write the string: + +```cpp +void write_json_file(const std::string& path, const json& j) +{ + const std::string text = j.dump(4) + '\n'; // may throw: nothing is written yet + + std::ofstream f(path, std::ios::binary); + if (!f.is_open()) + { + throw std::runtime_error("cannot open " + path); + } + f << text; + f.close(); + if (f.fail()) + { + throw std::runtime_error("cannot write " + path); + } +} +``` + +- **Serialize before opening the file.** Opening a `#!cpp std::ofstream` truncates the file. `dump` and + [`operator<<`](../api/operator_ltlt.md) throw [`type_error.316`](../home/exceptions.md#jsonexceptiontype_error316) + when a string is not valid UTF-8 (see [handling invalid UTF-8](serialization.md#handling-invalid-utf-8)), and + `#!cpp f << std::setw(4) << j` may already have written part of the value when it throws. Serializing into a string + first leaves the old file untouched in that case. +- **Check the stream after closing it.** A full disk or a missing permission sets the stream's error state; it does not + throw unless you enabled exceptions on the stream with `exceptions()`. +- `std::ios::binary` writes the text as it is. Without it, Windows writes `\r\n` for every newline of a pretty-printed + value. + +## Replacing a file without corrupting it + +The library never writes a file itself, so this section is not about the library. It is general C++ advice for a +problem users of the library keep running into. + +A program that crashes or is killed while it writes leaves a truncated file behind. To make sure readers see either the +old or the new contents, write to a temporary file in the same directory and rename it over the original +(C++17, `#!cpp `): + +```cpp +#include + +void replace_json_file(const std::filesystem::path& path, const json& j) +{ + std::filesystem::path tmp = path; + tmp += ".tmp"; // use a unique name if several processes may write the file + + write_json_file(tmp.string(), j); + std::filesystem::rename(tmp, path); // replaces path +} +``` + +The temporary file must be in the same directory (more precisely, on the same file system) as the target: a rename +across file systems does not fall back to a copy, it fails and throws `#!cpp std::filesystem::filesystem_error`. On +POSIX systems, the rename atomically replaces the file. Standard C++ has no way to flush a file to the disk (`fsync`), +so this protects against a crashed program, but not necessarily against a power loss. + +## Binary formats + +The [binary formats](binary_formats/index.md) work the same way, with the streams opened in binary mode: + +```cpp +// read +std::ifstream in("data.cbor", std::ios::binary); +json j = json::from_cbor(in); + +// write +const std::vector bytes = json::to_cbor(j); +std::ofstream out("data.cbor", std::ios::binary); +out.write(reinterpret_cast(bytes.data()), static_cast(bytes.size())); +``` + +Without `std::ios::binary`, Windows turns the bytes `0x0D 0x0A` into `0x0A` and may stop at a byte `0x1A` while reading, +and turns `0x0A` into `0x0D 0x0A` while writing, which corrupts the data. + +## See also + +- [Parsing](parsing/index.md) - the inputs `parse` accepts +- [Serialization](serialization.md) - the options of `dump` +- [`parse`](../api/basic_json/parse.md), [`dump`](../api/basic_json/dump.md) +- [`operator>>`](../api/operator_gtgt.md), [`operator<<`](../api/operator_ltlt.md) diff --git a/docs/mkdocs/docs/features/index.md b/docs/mkdocs/docs/features/index.md index f9b0f20cc..d8c6c9367 100644 --- a/docs/mkdocs/docs/features/index.md +++ b/docs/mkdocs/docs/features/index.md @@ -46,6 +46,8 @@ C++ types, and finally serialize it again. [BJData](binary_formats/bjdata.md), [BON8](binary_formats/bon8.md), [BSON](binary_formats/bson.md), [CBOR](binary_formats/cbor.md), [MessagePack](binary_formats/messagepack.md), or [UBJSON](binary_formats/ubjson.md). - [Binary values](binary_values.md) — store and exchange raw byte sequences. +- [Reading and writing files](files.md) — read and write files safely, including replacing a file without + corrupting it. ## How values are stored and configured diff --git a/docs/mkdocs/docs/features/json_patch.md b/docs/mkdocs/docs/features/json_patch.md index ea22b4e15..d9e96c831 100644 --- a/docs/mkdocs/docs/features/json_patch.md +++ b/docs/mkdocs/docs/features/json_patch.md @@ -29,7 +29,8 @@ other three continue on its result. [`patch`](../api/basic_json/patch.md) leaves the original value unchanged and returns the patched result as a copy. If the document is large and the original value is no longer needed, - [`patch_inplace`](../api/basic_json/patch_inplace.md) applies the same operations in place instead. + [`patch_inplace`](../api/basic_json/patch_inplace.md) applies the same operations in place instead. Unlike + `patch`, it is not atomic: if an operation fails, the operations before it stay applied. !!! note "Duplicate member names" diff --git a/docs/mkdocs/docs/features/json_view.md b/docs/mkdocs/docs/features/json_view.md index 44bf581a5..5ab2281f0 100644 --- a/docs/mkdocs/docs/features/json_view.md +++ b/docs/mkdocs/docs/features/json_view.md @@ -160,7 +160,9 @@ document: `#!cpp auto v = json_document::parse(text).root();` does not compile. wide. The limit is 16 bytes below 2^32, because the parser reads in 16-byte steps, and a position one step past the end must still fit. - **A stream is always read to its end.** There is no partial/streaming read of an `#!cpp std::istream`: the stream - is read to its end into a buffer the document owns, and parsing starts after that. + is read to its end into a buffer the document owns, and parsing starts after that. For a file, reading it into a + `#!cpp std::string` yourself and parsing that (an lvalue is borrowed) is therefore not more expensive in memory, and + was about 6 to 12 % faster than parsing from an `#!cpp std::ifstream`. - **Fewer source positions after edits.** With [`JSON_DIAGNOSTIC_POSITIONS`](../api/macros/json_diagnostic_positions.md) enabled, [`materialize()`](../api/basic_json_view/materialize.md) sets the same positions as `#!cpp basic_json::parse()`. diff --git a/docs/mkdocs/docs/features/parsing/index.md b/docs/mkdocs/docs/features/parsing/index.md index 901626952..93a02b501 100644 --- a/docs/mkdocs/docs/features/parsing/index.md +++ b/docs/mkdocs/docs/features/parsing/index.md @@ -35,6 +35,8 @@ std::ifstream f("example.json"); json data = json::parse(f); ``` +See [Reading and writing files](../files.md) for checking that the file was opened and for writing files safely. + The input must be encoded in UTF-8; other encodings are not supported. The exception is wide-character input, which is read as UTF-16 or UTF-32 — depending on the character type — and converted to UTF-8; see the [`parse`](../../api/basic_json/parse.md) page for an example. A single input may contain only one JSON value. @@ -84,6 +86,9 @@ ahead: after parsing, it is positioned exactly as if every character had been re [`operator>>`](../../api/operator_gtgt.md) can still read several values from one stream (see also [JSON Lines](json_lines.md)). +This does not hold for [`json_document`](../json_view.md): it reads a stream into a buffer it owns, so the stream saves +no memory, and reading the file into a `#!cpp std::string` yourself and borrowing it is about 6 to 12 % faster. + Two macros switch these optimizations off without changing any result: [`JSON_DISABLE_STREAM_BULK_SCAN`](../../api/macros/json_disable_stream_bulk_scan.md) for stream buffers that do not follow the usual `std::streambuf` contract, and diff --git a/docs/mkdocs/docs/features/performance.md b/docs/mkdocs/docs/features/performance.md index ffea9c215..8f4508057 100644 --- a/docs/mkdocs/docs/features/performance.md +++ b/docs/mkdocs/docs/features/performance.md @@ -39,7 +39,9 @@ iterator pair over non-contiguous but random-access storage (e.g. `#!cpp std::de optimization but not the second, since the byte-scanning fast path requires contiguous storage. Practically: a file can be passed directly to `parse`, which also avoids the memory for a copy of the file. Reading it -into a string first is no longer necessary for speed: +into a string first is no longer necessary for speed (this is for `#!cpp json::parse`; a +[`json_document`](json_view.md) reads a stream into an owned buffer anyway, so for it reading the file into a string +and borrowing that is the faster way): ```cpp // fast: the stream buffer's get area is scanned in bulk diff --git a/docs/mkdocs/docs/features/serialization.md b/docs/mkdocs/docs/features/serialization.md index c1b23442c..fd482cb73 100644 --- a/docs/mkdocs/docs/features/serialization.md +++ b/docs/mkdocs/docs/features/serialization.md @@ -17,6 +17,9 @@ To write a value directly to a stream (for example, a file or `#!cpp std::cout`) std::cout << j << std::endl; ``` +To write a file without leaving a truncated file behind when serialization fails, see +[Reading and writing files](files.md#writing-a-json-file). + !!! note "String, not raw value" `dump` always returns a **JSON text**. Serializing a JSON string therefore includes the surrounding quotes and diff --git a/docs/mkdocs/docs/home/faq.md b/docs/mkdocs/docs/home/faq.md index c43acef37..d06819569 100644 --- a/docs/mkdocs/docs/home/faq.md +++ b/docs/mkdocs/docs/home/faq.md @@ -110,7 +110,7 @@ in the source files that need it, instantiate `basic_json` once, and consider pr The parser did not see any character: the file was most likely not opened. A relative path is resolved against the working directory of the running program, which is often the build directory of your IDE, not the source directory. Check `is_open()` and try an absolute path. Other causes are an empty file and a stream that was already read. See -[`parse_error.101`](exceptions.md#jsonexceptionparse_error101). +[`parse_error.101`](exceptions.md#jsonexceptionparse_error101) and [Reading and writing files](../features/files.md). ### Input that arrives in chunks diff --git a/docs/mkdocs/mkdocs.yml b/docs/mkdocs/mkdocs.yml index 08c46ff83..ce5cdc226 100644 --- a/docs/mkdocs/mkdocs.yml +++ b/docs/mkdocs/mkdocs.yml @@ -130,6 +130,7 @@ nav: - features/element_access/unchecked_access.md - features/element_access/checked_access.md - features/element_access/default_value.md + - features/files.md - features/iterators.md - features/merge_patch.md - features/json_patch.md From 1da873515416639583b7c6143646bbf064431657 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20Gr=C3=BCninger?= Date: Wed, 4 Nov 2026 23:38:47 +0100 Subject: [PATCH 47/54] Fix leak and crash when an allocation throws during parsing (#6023) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit json_sax_dom_parser::insert_key_with_index: own the new object_key_index in a unique_ptr before key_indexes may reallocate; a throwing reallocation leaked it. Signed-off-by: Christoph Grüninger --- include/nlohmann/detail/input/json_sax.hpp | 3 ++- single_include/nlohmann/json.hpp | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/include/nlohmann/detail/input/json_sax.hpp b/include/nlohmann/detail/input/json_sax.hpp index 80850ab92..fc9e9b774 100644 --- a/include/nlohmann/detail/input/json_sax.hpp +++ b/include/nlohmann/detail/input/json_sax.hpp @@ -904,7 +904,8 @@ class json_sax_dom_parser : public json_sax_dom_error_state { hashes.push_back(hasher(element.first)); } - ordered_data.key_indexes.emplace_back(new object_key_index(std::move(hashes))); + std::unique_ptr index(new object_key_index(std::move(hashes))); // NOLINT(cppcoreguidelines-owning-memory,modernize-make-unique): std::make_unique is C++14 + ordered_data.key_indexes.push_back(std::move(index)); f.ordered.index = ordered_data.key_indexes.back().get(); } diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index bb81909ae..50089514f 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -16538,7 +16538,8 @@ class json_sax_dom_parser : public json_sax_dom_error_state { hashes.push_back(hasher(element.first)); } - ordered_data.key_indexes.emplace_back(new object_key_index(std::move(hashes))); + std::unique_ptr index(new object_key_index(std::move(hashes))); // NOLINT(cppcoreguidelines-owning-memory,modernize-make-unique): std::make_unique is C++14 + ordered_data.key_indexes.push_back(std::move(index)); f.ordered.index = ordered_data.key_indexes.back().get(); } From 20fe34ceeb58a3c1ba3750258c56fb231d778762 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 4 Nov 2026 23:38:57 +0100 Subject: [PATCH 48/54] build(deps): bump lukka/get-cmake from 4.4.4 to 4.4.5 (#6024) Bumps [lukka/get-cmake](https://github.com/lukka/get-cmake) from 4.4.4 to 4.4.5. - [Release notes](https://github.com/lukka/get-cmake/releases) - [Changelog](https://github.com/lukka/get-cmake/blob/main/RELEASE_PROCESS.md) - [Commits](https://github.com/lukka/get-cmake/compare/a89e43e22a96abafcd7a62db652a65ecd33052c9...f86dd60acae0d132056913e248bdabf0e0580b7a) --- updated-dependencies: - dependency-name: lukka/get-cmake dependency-version: 4.4.5 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ubuntu.yml | 36 +++++++++++++++++------------------ .github/workflows/windows.yml | 4 ++-- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 5dd479bcd..581fb6081 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -25,7 +25,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -38,7 +38,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Check that Meson and pkg-config offer the CMake options run: make check_build_options - name: Install Meson @@ -92,7 +92,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -123,7 +123,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -142,7 +142,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -168,7 +168,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -244,7 +244,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Run CMake run: CXX=g++-${{ matrix.compiler }} cmake -S . -B build -DJSON_CI=On - name: Build @@ -262,7 +262,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -279,7 +279,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Disable filesystem support (std::filesystem is unusable) for clang 7 / 8 / 9 / 10 run: echo "JSON_FORCED_GLOBAL_COMPILE_OPTIONS=-DJSON_HAS_FILESYSTEM=0;-DJSON_HAS_EXPERIMENTAL_FILESYSTEM=0" >> "$GITHUB_ENV" if: ${{ matrix.compiler == '7' || matrix.compiler == '8' || matrix.compiler == '9' || matrix.compiler == '10' }} @@ -299,7 +299,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -319,7 +319,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build with libc++ @@ -346,7 +346,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -366,7 +366,7 @@ jobs: # import-std support. Its opt-in token is CMake-version-specific, so pin # CMake to the version whose token is set in tests/module_cpp20/CMakeLists.txt. - name: Get pinned CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 with: cmakeVersion: 4.3.4 # Clang: the std library module is provided by libc++ (the image's libstdc++ @@ -392,7 +392,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -409,7 +409,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -423,7 +423,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -443,7 +443,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Run CMake run: cmake -S . -B build -DCMAKE_TOOLCHAIN_FILE=$EMSDK/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake -GNinja - name: Build @@ -484,7 +484,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Check that the job runs natively on arm64 run: uname -m && test "$(uname -m)" = aarch64 - name: Run CMake diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 59647ed94..b9f9d0251 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -90,7 +90,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Set extra CXX_FLAGS for latest std_version # /wd5285 silences C5285 emitted by the bundled third-party doctest.h, which # specializes std::tuple (newly diagnosed by the VS2026 v145 toolset) @@ -216,7 +216,7 @@ jobs: # import-std support. Its opt-in token is CMake-version-specific, so pin # CMake to the version whose token is set in tests/module_cpp20/CMakeLists.txt. - name: Get pinned CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 with: cmakeVersion: 4.3.4 # CMAKE_CXX_FLAGS replaces CMake's MSVC defaults (which include /EHsc), and From e48800f09d6f5e31320371da7ebdda7f7c6e911e Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Wed, 4 Nov 2026 23:42:17 +0100 Subject: [PATCH 49/54] Binary reader and writer: remove checks that invariants make dead (#6001) * Binary reader and writer: remove checks that invariants make dead Branch coverage of the test suite and of 24k mutated inputs showed these conditions as never taken; each is implied by a check made earlier on the same path: - check_bson_document_size: `document_size < 0` (the size was rejected below 5 before it was stored) - get_ubjson_ndarray_size: `second != 'N'` ('N' is rejected as a type marker before `second` can be set) - bjd_type_name() == nullptr for an ndarray dtype: every marker is_valid_optimized_type() accepts for BJData has a name (now an assert) - the early is_bjd_excluded_optimized_type() check in get_ubjson_size_type(): is_valid_optimized_type() rejects the same eight markers with the same error - `input_format == bjdata` next to `is_ndarray`, bit 8 of the type, or a 'B'/'u'/'m'/'M' marker: these only arise for BJData - `first != npos` next to bit 8: the bit is only set after a count - MessagePack writer: the `<= max` halves of the negative int8/16/32 ladder (the ladder is in the `else` of `n >= 0`) No change in behaviour: a build that aborts on each removed side passes the test suite and the fuzz corpora, and a 1.6M-input differential run of the reader against the original headers found no difference. Signed-off-by: Niels Lohmann * Remove is_bjd_excluded_optimized_type(), which has no caller left; fix two comments is_valid_optimized_type() rejects the markers BJData excludes, so the reader no longer needs its own list; the test now checks is_valid_optimized_type() of a BJData reader against the writer's list. The BSON size comment now says that open_bson_document() rejects a size below 5. Signed-off-by: Niels Lohmann * Make the BJData reader in the lookup test const (misc-const-correctness) Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann --- .../nlohmann/detail/input/binary_reader.hpp | 65 +++++----------- .../nlohmann/detail/output/binary_writer.hpp | 11 +-- single_include/nlohmann/json.hpp | 76 ++++++------------- tests/src/unit-bjdata.cpp | 16 ++-- 4 files changed, 57 insertions(+), 111 deletions(-) diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index b793ae097..93f707097 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -344,8 +344,8 @@ class binary_reader nested document could declare a length that disagrees with where its terminator actually falls and quietly hand the bytes in between to the enclosing document. A well-formed document is at least 5 bytes (the prefix - plus the terminator); the equality also rejects those impossible sizes, - since at least 5 bytes are always consumed. + plus the terminator); open_bson_document() rejects a smaller size before + it is stored, so @a document_size is never negative here. @param[in] document_start value of chars_read before the size prefix @param[in] document_size the declared document size @@ -353,7 +353,7 @@ class binary_reader */ bool check_bson_document_size(const std::size_t document_start, const std::int32_t document_size) { - if (JSON_HEDLEY_UNLIKELY(document_size < 0 || static_cast(document_size) != chars_read - document_start)) + if (JSON_HEDLEY_UNLIKELY(static_cast(document_size) != chars_read - document_start)) { return bson_document_size_error(document_size, chars_read - document_start); } @@ -3094,16 +3094,13 @@ class binary_reader { if (size_and_type.second != 0) { - if (size_and_type.second != 'N') + for (std::size_t i = 0; i < size_and_type.first; ++i) { - for (std::size_t i = 0; i < size_and_type.first; ++i) + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_value(dimlen, no_ndarray, size_and_type.second))) { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_value(dimlen, no_ndarray, size_and_type.second))) - { - return false; - } - dim.push_back(dimlen); + return false; } + dim.push_back(dimlen); } } else @@ -3323,11 +3320,9 @@ class binary_reader // follow the documented _ArrayType_, _ArraySize_, _ArrayData_ order if (ndarray_dtype != 0) { + // every optimized type marker is_valid_optimized_type() accepts for BJData has a name const char* type_name = bjd_type_name(ndarray_dtype); - if (JSON_HEDLEY_UNLIKELY(type_name == nullptr)) - { - return unexpected_byte("invalid byte", "type"); - } + JSON_ASSERT(type_name != nullptr); string_t type_key = "_ArrayType_"; string_t type = type_name; // sax->string() takes a reference @@ -3410,12 +3405,6 @@ class binary_reader if (current == '$') { result.second = get(); // must not ignore 'N', because 'N' maybe the type - if (input_format == input_format_t::bjdata - && JSON_HEDLEY_UNLIKELY(is_bjd_excluded_optimized_type(result.second))) - { - return optimized_type_error(); - } - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof("type"))) { return false; @@ -3443,7 +3432,7 @@ class binary_reader // an ndarray was read here only if the flag flipped; when it was // seeded true, get_ubjson_size_value() already rejected the nested // dimension vector - if (input_format == input_format_t::bjdata && is_ndarray && !inside_ndarray) + if (is_ndarray && !inside_ndarray) { result.second |= (1 << 8); // use bit 8 to indicate ndarray, all UBJSON and BJData markers should be ASCII letters } @@ -3453,7 +3442,7 @@ class binary_reader if (current == '#') { const bool is_error = get_ubjson_size_value(result.first, is_ndarray); - if (input_format == input_format_t::bjdata && is_ndarray && !inside_ndarray) + if (is_ndarray && !inside_ndarray) { return last_byte_error(exception_id::unexpected_byte, "ndarray requires both type and size", "size"); } @@ -3627,7 +3616,6 @@ class binary_reader JSON_HEDLEY_NEVER_INLINE bool parse_ubjson_typed_run(std::size_t& remaining, const char_int_type type, std::true_type /*bulk*/) { - const bool bjdata = input_format == input_format_t::bjdata; switch (type) { case 'U': @@ -3641,11 +3629,11 @@ class binary_reader case 'L': return parse_ubjson_typed_run(remaining); case 'u': - return !bjdata || parse_ubjson_typed_run(remaining); + return parse_ubjson_typed_run(remaining); case 'm': - return !bjdata || parse_ubjson_typed_run(remaining); + return parse_ubjson_typed_run(remaining); case 'M': - return !bjdata || parse_ubjson_typed_run(remaining); + return parse_ubjson_typed_run(remaining); case 'd': return parse_ubjson_typed_run(remaining); case 'D': @@ -3726,7 +3714,7 @@ class binary_reader // if bit-8 of size_and_type.second is set to 1, encode bjdata ndarray as an object in JData annotated array format (https://github.com/NeuroJSON/jdata): // {"_ArrayType_" : "typeid", "_ArraySize_" : [n1, n2, ...], "_ArrayData_" : [v1, v2, ...]} - if (input_format == input_format_t::bjdata && size_and_type.first != npos && (size_and_type.second & (1 << 8)) != 0) + if ((size_and_type.second & (1 << 8)) != 0) { size_and_type.second &= ~(static_cast(1) << 8); // use bit 8 to indicate ndarray, here we remove the bit to restore the type marker @@ -3785,7 +3773,7 @@ class binary_reader } // If BJData type marker is 'B' decode as binary - if (input_format == input_format_t::bjdata && size_and_type.first != npos && size_and_type.second == 'B') + if (size_and_type.first != npos && size_and_type.second == 'B') { binary_t result; return get_binary(size_and_type.first, result) && sax->binary(result); @@ -3834,7 +3822,7 @@ class binary_reader } // do not accept ND-array size in objects in BJData - if (input_format == input_format_t::bjdata && size_and_type.first != npos && (size_and_type.second & (1 << 8)) != 0) + if ((size_and_type.second & (1 << 8)) != 0) { return last_byte_error(exception_id::unexpected_byte, "BJData object does not support ND-array size in optimized format", "object"); } @@ -5115,8 +5103,9 @@ class binary_reader JSON_PRIVATE_UNLESS_TESTED: /*! @brief check whether @a marker is a type marker that may follow '$' - UBJSON: Z N T F i U I l L d D H C S [ {; BJData: the numeric markers and C - (BJData's excluded markers are rejected separately). + UBJSON: Z N T F i U I l L d D H C S [ {; BJData: the numeric markers and C. + The markers BJData excludes (F H N S T Z [ {) are the ones + binary_writer::is_bjdata_excluded_type_marker() lists; keep the two in sync. */ bool is_valid_optimized_type(const char_int_type marker) const noexcept { @@ -5151,20 +5140,6 @@ class binary_reader } } - /*! - @brief whether @a marker is excluded from BJData's optimized ND-array types - @return whether @a marker is one of 'F', 'H', 'N', 'S', 'T', 'Z', '[', '{' - - Mirrors binary_writer's @ref binary_writer::is_bjdata_excluded_type_marker - "is_bjdata_excluded_type_marker()`, which encodes the same list the other - way; keep the two in sync. - */ - static constexpr bool is_bjd_excluded_optimized_type(const char_int_type marker) noexcept - { - return marker == '[' || marker == '{' || marker == 'S' || marker == 'H' - || marker == 'T' || marker == 'F' || marker == 'N' || marker == 'Z'; - } - /*! @brief look up the ND-array element type name for a BJData dtype marker @return the type name ("uint8", "int8", ...), or nullptr if @a marker does diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 5bbaa42d2..dda9c3758 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -3126,22 +3126,19 @@ class binary_writer // negative fixnum write_number(static_cast(j.m_data.m_value.number_integer)); } - else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)() && - j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) + else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)()) { // int 8 oa.write_character(to_char_type(0xD0)); write_number(static_cast(j.m_data.m_value.number_integer)); } - else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)() && - j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) + else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)()) { // int 16 oa.write_character(to_char_type(0xD1)); write_number(static_cast(j.m_data.m_value.number_integer)); } - else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)() && - j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) + else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)()) { // int 32 oa.write_character(to_char_type(0xD2)); @@ -4057,7 +4054,7 @@ class binary_writer Containers, strings, high-precision numbers, booleans and null cannot be declared as the single type of an optimized container in BJData; such a container is written unoptimized. The reader rejects them with the same - list (binary_reader::is_bjd_excluded_optimized_type()). + list (binary_reader::is_valid_optimized_type()). */ static constexpr bool is_bjdata_excluded_type_marker(const CharType marker) noexcept { diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 50089514f..eedb9c561 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -18360,8 +18360,8 @@ class binary_reader nested document could declare a length that disagrees with where its terminator actually falls and quietly hand the bytes in between to the enclosing document. A well-formed document is at least 5 bytes (the prefix - plus the terminator); the equality also rejects those impossible sizes, - since at least 5 bytes are always consumed. + plus the terminator); open_bson_document() rejects a smaller size before + it is stored, so @a document_size is never negative here. @param[in] document_start value of chars_read before the size prefix @param[in] document_size the declared document size @@ -18369,7 +18369,7 @@ class binary_reader */ bool check_bson_document_size(const std::size_t document_start, const std::int32_t document_size) { - if (JSON_HEDLEY_UNLIKELY(document_size < 0 || static_cast(document_size) != chars_read - document_start)) + if (JSON_HEDLEY_UNLIKELY(static_cast(document_size) != chars_read - document_start)) { return bson_document_size_error(document_size, chars_read - document_start); } @@ -21110,16 +21110,13 @@ class binary_reader { if (size_and_type.second != 0) { - if (size_and_type.second != 'N') + for (std::size_t i = 0; i < size_and_type.first; ++i) { - for (std::size_t i = 0; i < size_and_type.first; ++i) + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_value(dimlen, no_ndarray, size_and_type.second))) { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_value(dimlen, no_ndarray, size_and_type.second))) - { - return false; - } - dim.push_back(dimlen); + return false; } + dim.push_back(dimlen); } } else @@ -21339,11 +21336,9 @@ class binary_reader // follow the documented _ArrayType_, _ArraySize_, _ArrayData_ order if (ndarray_dtype != 0) { + // every optimized type marker is_valid_optimized_type() accepts for BJData has a name const char* type_name = bjd_type_name(ndarray_dtype); - if (JSON_HEDLEY_UNLIKELY(type_name == nullptr)) - { - return unexpected_byte("invalid byte", "type"); - } + JSON_ASSERT(type_name != nullptr); string_t type_key = "_ArrayType_"; string_t type = type_name; // sax->string() takes a reference @@ -21426,12 +21421,6 @@ class binary_reader if (current == '$') { result.second = get(); // must not ignore 'N', because 'N' maybe the type - if (input_format == input_format_t::bjdata - && JSON_HEDLEY_UNLIKELY(is_bjd_excluded_optimized_type(result.second))) - { - return optimized_type_error(); - } - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof("type"))) { return false; @@ -21459,7 +21448,7 @@ class binary_reader // an ndarray was read here only if the flag flipped; when it was // seeded true, get_ubjson_size_value() already rejected the nested // dimension vector - if (input_format == input_format_t::bjdata && is_ndarray && !inside_ndarray) + if (is_ndarray && !inside_ndarray) { result.second |= (1 << 8); // use bit 8 to indicate ndarray, all UBJSON and BJData markers should be ASCII letters } @@ -21469,7 +21458,7 @@ class binary_reader if (current == '#') { const bool is_error = get_ubjson_size_value(result.first, is_ndarray); - if (input_format == input_format_t::bjdata && is_ndarray && !inside_ndarray) + if (is_ndarray && !inside_ndarray) { return last_byte_error(exception_id::unexpected_byte, "ndarray requires both type and size", "size"); } @@ -21643,7 +21632,6 @@ class binary_reader JSON_HEDLEY_NEVER_INLINE bool parse_ubjson_typed_run(std::size_t& remaining, const char_int_type type, std::true_type /*bulk*/) { - const bool bjdata = input_format == input_format_t::bjdata; switch (type) { case 'U': @@ -21657,11 +21645,11 @@ class binary_reader case 'L': return parse_ubjson_typed_run(remaining); case 'u': - return !bjdata || parse_ubjson_typed_run(remaining); + return parse_ubjson_typed_run(remaining); case 'm': - return !bjdata || parse_ubjson_typed_run(remaining); + return parse_ubjson_typed_run(remaining); case 'M': - return !bjdata || parse_ubjson_typed_run(remaining); + return parse_ubjson_typed_run(remaining); case 'd': return parse_ubjson_typed_run(remaining); case 'D': @@ -21742,7 +21730,7 @@ class binary_reader // if bit-8 of size_and_type.second is set to 1, encode bjdata ndarray as an object in JData annotated array format (https://github.com/NeuroJSON/jdata): // {"_ArrayType_" : "typeid", "_ArraySize_" : [n1, n2, ...], "_ArrayData_" : [v1, v2, ...]} - if (input_format == input_format_t::bjdata && size_and_type.first != npos && (size_and_type.second & (1 << 8)) != 0) + if ((size_and_type.second & (1 << 8)) != 0) { size_and_type.second &= ~(static_cast(1) << 8); // use bit 8 to indicate ndarray, here we remove the bit to restore the type marker @@ -21801,7 +21789,7 @@ class binary_reader } // If BJData type marker is 'B' decode as binary - if (input_format == input_format_t::bjdata && size_and_type.first != npos && size_and_type.second == 'B') + if (size_and_type.first != npos && size_and_type.second == 'B') { binary_t result; return get_binary(size_and_type.first, result) && sax->binary(result); @@ -21850,7 +21838,7 @@ class binary_reader } // do not accept ND-array size in objects in BJData - if (input_format == input_format_t::bjdata && size_and_type.first != npos && (size_and_type.second & (1 << 8)) != 0) + if ((size_and_type.second & (1 << 8)) != 0) { return last_byte_error(exception_id::unexpected_byte, "BJData object does not support ND-array size in optimized format", "object"); } @@ -23131,8 +23119,9 @@ class binary_reader JSON_PRIVATE_UNLESS_TESTED: /*! @brief check whether @a marker is a type marker that may follow '$' - UBJSON: Z N T F i U I l L d D H C S [ {; BJData: the numeric markers and C - (BJData's excluded markers are rejected separately). + UBJSON: Z N T F i U I l L d D H C S [ {; BJData: the numeric markers and C. + The markers BJData excludes (F H N S T Z [ {) are the ones + binary_writer::is_bjdata_excluded_type_marker() lists; keep the two in sync. */ bool is_valid_optimized_type(const char_int_type marker) const noexcept { @@ -23167,20 +23156,6 @@ class binary_reader } } - /*! - @brief whether @a marker is excluded from BJData's optimized ND-array types - @return whether @a marker is one of 'F', 'H', 'N', 'S', 'T', 'Z', '[', '{' - - Mirrors binary_writer's @ref binary_writer::is_bjdata_excluded_type_marker - "is_bjdata_excluded_type_marker()`, which encodes the same list the other - way; keep the two in sync. - */ - static constexpr bool is_bjd_excluded_optimized_type(const char_int_type marker) noexcept - { - return marker == '[' || marker == '{' || marker == 'S' || marker == 'H' - || marker == 'T' || marker == 'F' || marker == 'N' || marker == 'Z'; - } - /*! @brief look up the ND-array element type name for a BJData dtype marker @return the type name ("uint8", "int8", ...), or nullptr if @a marker does @@ -31144,22 +31119,19 @@ class binary_writer // negative fixnum write_number(static_cast(j.m_data.m_value.number_integer)); } - else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)() && - j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) + else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)()) { // int 8 oa.write_character(to_char_type(0xD0)); write_number(static_cast(j.m_data.m_value.number_integer)); } - else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)() && - j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) + else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)()) { // int 16 oa.write_character(to_char_type(0xD1)); write_number(static_cast(j.m_data.m_value.number_integer)); } - else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)() && - j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) + else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)()) { // int 32 oa.write_character(to_char_type(0xD2)); @@ -32075,7 +32047,7 @@ class binary_writer Containers, strings, high-precision numbers, booleans and null cannot be declared as the single type of an optimized container in BJData; such a container is written unoptimized. The reader rejects them with the same - list (binary_reader::is_bjd_excluded_optimized_type()). + list (binary_reader::is_valid_optimized_type()). */ static constexpr bool is_bjdata_excluded_type_marker(const CharType marker) noexcept { diff --git a/tests/src/unit-bjdata.cpp b/tests/src/unit-bjdata.cpp index ecc79959d..dd5831dbd 100644 --- a/tests/src/unit-bjdata.cpp +++ b/tests/src/unit-bjdata.cpp @@ -78,23 +78,25 @@ TEST_CASE("BJData") { SECTION("binary_reader BJData lookup tables") { - // both lookups are static member functions + // bjd_type_name() is static; is_valid_optimized_type() needs a BJData reader std::vector const data; using reader_t = nlohmann::detail::binary_reader; - // the excluded optimized-type markers must match binary_writer's - // is_bjdata_excluded_type_marker(), which encodes the same 8 markers + // the markers BJData excludes as optimized types must match + // binary_writer's is_bjdata_excluded_type_marker(), which encodes the + // same 8 markers + const reader_t reader(nlohmann::detail::input_adapter(data), json::input_format_t::bjdata); for (const char marker : - {'[', '{', 'S', 'H', 'T', 'F', 'N', 'Z' + {'[', '{', 'S', 'H', 'T', 'F', 'N', 'Z', 'x' }) { - CHECK(reader_t::is_bjd_excluded_optimized_type(static_cast(marker))); + CHECK(!reader.is_valid_optimized_type(static_cast(marker))); } for (const char marker : - {'U', 'i', 'u', 'I', 'm', 'l', 'M', 'L', 'd', 'D', 'C', 'B', 'x' + {'U', 'i', 'u', 'I', 'm', 'l', 'M', 'L', 'd', 'D', 'C', 'B', 'h' }) { - CHECK(!reader_t::is_bjd_excluded_optimized_type(static_cast(marker))); + CHECK(reader.is_valid_optimized_type(static_cast(marker))); } // every dtype marker must round-trip to its ND-array type name From b54267d2a74c085ff2b05b01e369ecb7dd70cec6 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Thu, 5 Nov 2026 16:04:39 +0100 Subject: [PATCH 50/54] Docs: compile times of JSON_HAS_STD_FORMAT=0 and JSON_HAS_RANGES=0 (#6025) Add sections on switching off std::format and ranges support to the compile-times page, with measurements (Apple clang, GCC 16, -std=c++20), the functionality that is lost, and chart rows (issue #5294). Signed-off-by: Niels Lohmann --- docs/mkdocs/docs/images/compile_times.svg | 56 ++++++----- docs/mkdocs/docs/integration/compile_times.md | 92 ++++++++++++++++++- docs/mkdocs/scripts/figures/charts.py | 2 + 3 files changed, 126 insertions(+), 24 deletions(-) diff --git a/docs/mkdocs/docs/images/compile_times.svg b/docs/mkdocs/docs/images/compile_times.svg index e8e904a60..302817e1c 100644 --- a/docs/mkdocs/docs/images/compile_times.svg +++ b/docs/mkdocs/docs/images/compile_times.svg @@ -1,21 +1,21 @@ - + Change in compile time against the default configuration at -O0, Apple clang and G​CC 16 -Grouped horizontal bars, negative is faster. json_fwd.hpp instead of json.hpp on a header-only translation unit: Apple clang -62 %, G​CC 16 -82 %; extern template on a parsing translation unit: Apple clang -7 %, G​CC 16 -21 %; JSON_NO_AUTOMATIC_UDLS on a model translation unit: Apple clang -11 %, G​CC 16 -11 %; JSON_NO_AUTOMATIC_UDLS on a parsing translation unit: Apple clang -1 %, G​CC 16 -2 %; JSON_HAS_FILESYSTEM=0 + JSON_NO_IO on a model translation unit: Apple clang -8 %, G​CC 16 -11 %; JSON_HAS_FILESYSTEM=0 on a model translation unit: Apple clang -5 %, G​CC 16 -9 %; JSON_NO_IO on a model translation unit: Apple clang -2 %, G​CC 16 -2 %; JSON_USE_GLOBAL_UDLS=0 on a model translation unit: Apple clang 0 %, G​CC 16 -1 %. +Grouped horizontal bars, negative is faster. json_fwd.hpp instead of json.hpp on a header-only translation unit: Apple clang -62 %, G​CC 16 -82 %; extern template on a parsing translation unit: Apple clang -7 %, G​CC 16 -21 %; JSON_NO_AUTOMATIC_UDLS on a model translation unit: Apple clang -11 %, G​CC 16 -11 %; JSON_NO_AUTOMATIC_UDLS on a parsing translation unit: Apple clang -1 %, G​CC 16 -2 %; JSON_HAS_FILESYSTEM=0 + JSON_NO_IO on a model translation unit: Apple clang -8 %, G​CC 16 -11 %; JSON_HAS_FILESYSTEM=0 on a model translation unit: Apple clang -5 %, G​CC 16 -9 %; JSON_HAS_STD_FORMAT=0 on a model translation unit: Apple clang -1 %, G​CC 16 -4 %; JSON_HAS_RANGES=0 on a model translation unit: Apple clang -4 %, G​CC 16 -1 %; JSON_NO_IO on a model translation unit: Apple clang -2 %, G​CC 16 -2 %; JSON_USE_GLOBAL_UDLS=0 on a model translation unit: Apple clang 0 %, G​CC 16 -1 %. Apple clang G​CC 16 - -−80 % - -−60 % - -−40 % - -−20 % -0 % -← faster -slower → + +−80 % + +−60 % + +−40 % + +−20 % +0 % +← faster +slower → json_fwd.hpp instead of json.hpp · header-only TU json_fwd.hpp instead of json.hpp, header-only TU (Apple clang): −62 % −62 % @@ -46,15 +46,25 @@ −5 % JSON_HAS_FILESYSTEM=0, model TU (G​CC 16): −9 % −9 % -JSON_NO_IO · model TU* -JSON_NO_IO, model TU (Apple clang): −2 % -−2 % -JSON_NO_IO, model TU (G​CC 16): −2 % -−2 % -JSON_USE_GLOBAL_UDLS=0 · model TU* -JSON_USE_GLOBAL_UDLS=0, model TU (Apple clang): 0 % -0 % -JSON_USE_GLOBAL_UDLS=0, model TU (G​CC 16): −1 % +JSON_HAS_STD_FORMAT=0 · model TU +JSON_HAS_STD_FORMAT=0, model TU (Apple clang): −1 % +−1 % +JSON_HAS_STD_FORMAT=0, model TU (G​CC 16): −4 % +−4 % +JSON_HAS_RANGES=0 · model TU +JSON_HAS_RANGES=0, model TU (Apple clang): −4 % +−4 % +JSON_HAS_RANGES=0, model TU (G​CC 16): −1 % −1 % - +JSON_NO_IO · model TU* +JSON_NO_IO, model TU (Apple clang): −2 % +−2 % +JSON_NO_IO, model TU (G​CC 16): −2 % +−2 % +JSON_USE_GLOBAL_UDLS=0 · model TU* +JSON_USE_GLOBAL_UDLS=0, model TU (Apple clang): 0 % +0 % +JSON_USE_GLOBAL_UDLS=0, model TU (G​CC 16): −1 % +−1 % + diff --git a/docs/mkdocs/docs/integration/compile_times.md b/docs/mkdocs/docs/integration/compile_times.md index 8798931cb..699b21f05 100644 --- a/docs/mkdocs/docs/integration/compile_times.md +++ b/docs/mkdocs/docs/integration/compile_times.md @@ -11,6 +11,11 @@ that cost, ordered by how much they typically save. 16.2.0 on macOS (Apple M1 Max). All configurations were run interleaved, in alternating order. They show the order of magnitude to expect; measure your own code before and after a change. + The two sections on `std::format` and ranges apply to C++20 only and were measured with `-std=c++20`, so their + baselines are slower than the others. Their differences are mostly within the run-to-run noise of a few percent: + two complete runs of the same configuration differed by up to 10% in the median. The tables show the later run (one + outlier cell was repeated). + The translation units are listed below. They replace the ones used for earlier versions of this page, whose sources were not published, so the numbers differ from earlier versions. @@ -66,7 +71,8 @@ that cost, ordered by how much they typically save. --8<-- "images/compile_times.svg"

Change in compile time at `-O0` against the default configuration of the same -translation unit, taken from the tables below (medians of nine runs, see above). * computed from the milliseconds in +translation unit, taken from the tables below (medians of nine runs, see above; the `std::format` and ranges rows are +measured with `-std=c++20`). * computed from the milliseconds in [Options without effect on compile times](#options-without-effect-on-compile-times).

## Include `json_fwd.hpp` in headers @@ -214,6 +220,88 @@ from and to `std::string`; whether that fallback exists depends on the platform' (`std::wstring` on Windows), so check code that converts paths. `JSON_NO_IO` additionally removes all stream functions (see [Options without effect](#options-without-effect-on-compile-times)). +## Switch off `std::format` support + +With C++20 and a standard library that provides ``, the library includes it to provide a +[`std::formatter`](../api/basic_json/std_formatter.md) specialization (see +[`JSON_HAS_STD_FORMAT`](../api/macros/json_has_std_format.md)). If you do not format JSON values with `std::format`, +switch the detection off for the whole project: + +```cmake +target_compile_definitions(my_target PRIVATE JSON_HAS_STD_FORMAT=0) +``` + +| Compiler | Translation unit | Default (`-O0` / `-O2`) | `std::format` off (`-O0` / `-O2`) | Change | +|-------------|------------------|------------------------:|----------------------------------:|-----------:| +| Apple clang | model | 1283 ms / 1544 ms | 1269 ms / 1539 ms | −1% / 0% | +| GCC 16 | model | 2026 ms / 2482 ms | 1944 ms / 2416 ms | −4% / −3% | +| Apple clang | parsing | 1350 ms / 2278 ms | 1347 ms / 2300 ms | 0% / +1% | +| GCC 16 | parsing | 2715 ms / 4351 ms | 2644 ms / 4287 ms | −3% / −1% | + +

Compile time with `JSON_HAS_STD_FORMAT=0` against the default configuration, both with `-std=c++20`, at `-O0` and `-O2`.

+ +The saving is small, and only GCC shows it: libstdc++'s `` is an additional header that nothing else in the +translation unit includes, so the macro removes it (and the headers it pulls in). With libc++ (Apple clang), other +standard headers include `` anyway, so the macro only removes the library's own formatter, and the difference is +within the noise. + +The trade-off: the `std::formatter` specialization is removed, so `#!cpp std::format("{}", j)`, +and `#!cpp std::format_to` with a JSON value no longer compile (in a test with both compilers on +macOS, the error was that `std::formatter` must be specialized for the type). Use [`dump()`](../api/basic_json/dump.md), +or the stream operators, instead. + +## Switch off ranges support + +With C++20 and a standard library that provides ``, the library includes it to declare the +[`items()`](../api/basic_json/items.md) proxy a *borrowed range*, and uses the `std::ranges` concepts and algorithms for +construction from range views and for iterator input (see [`JSON_HAS_RANGES`](../api/macros/json_has_ranges.md)). If you +do not need these, switch the detection off for the whole project: + +```cmake +target_compile_definitions(my_target PRIVATE JSON_HAS_RANGES=0) +``` + +| Compiler | Translation unit | Default (`-O0` / `-O2`) | Ranges off (`-O0` / `-O2`) | Change | +|-------------|------------------|------------------------:|---------------------------:|-----------:| +| Apple clang | model | 1283 ms / 1544 ms | 1237 ms / 1495 ms | −4% / −3% | +| GCC 16 | model | 2026 ms / 2482 ms | 2004 ms / 2464 ms | −1% / −1% | +| Apple clang | parsing | 1350 ms / 2278 ms | 1311 ms / 2284 ms | −3% / 0% | +| GCC 16 | parsing | 2715 ms / 4351 ms | 2705 ms / 4427 ms | 0% / +2% | + +

Compile time with `JSON_HAS_RANGES=0` against the default configuration, both with `-std=c++20`, at `-O0` and `-O2`.

+ +The saving is small as well, and with GCC it is not distinguishable from the noise: its `` includes ``, +so with `std::format` support left on, part of the ranges machinery is still parsed. +Both macros together save more than either alone: + +```cmake +target_compile_definitions(my_target PRIVATE JSON_HAS_STD_FORMAT=0 JSON_HAS_RANGES=0) +``` + +| Compiler | Translation unit | Default (`-O0` / `-O2`) | Both off (`-O0` / `-O2`) | Change | +|-------------|------------------|------------------------:|-------------------------:|-----------:| +| Apple clang | model | 1283 ms / 1544 ms | 1223 ms / 1504 ms | −5% / −3% | +| GCC 16 | model | 2026 ms / 2482 ms | 1914 ms / 2378 ms | −6% / −4% | +| Apple clang | parsing | 1350 ms / 2278 ms | 1311 ms / 2271 ms | −3% / 0% | +| GCC 16 | parsing | 2715 ms / 4351 ms | 2635 ms / 4260 ms | −3% / −2% | + +

Compile time with `JSON_HAS_STD_FORMAT=0` and `JSON_HAS_RANGES=0` against the default configuration, both with `-std=c++20`, at `-O0` and `-O2`.

+ +The trade-offs of `JSON_HAS_RANGES=0` are larger than those of the other options on this page: + +- `items()` is no longer a borrowed range. On a temporary, `#!cpp std::ranges::begin(make_json().items())` no longer + compiles, and `std::ranges` algorithms such as `#!cpp std::ranges::find_if(make_json().items(), pred)` return + `std::ranges::dangling` instead of an iterator, so code that uses the result no longer compiles. Calling them on + `#!cpp j.items()` for a named `j` is not affected. +- [`JSON_HAS_RANGE_VIEW_CONVERSION`](../api/macros/json_has_range_view_conversion.md) becomes `0`, so a JSON array can no + longer be constructed from a `std::ranges::view` such as the result of `std::views::filter`. The assignment + `#!cpp json j = v | std::views::filter(pred);` no longer compiles. +- Parsing from an iterator pair loses the C++20 fast path for contiguous iterators that are not pointers (for + example those of `std::vector`) and falls back to byte-at-a-time scanning. The result is the same; parsing may be + slower (not measured here). + +The first two were checked with both compilers on macOS; the third is read from the library's source. + ## Use C++20 modules With a toolchain that supports named modules, `import nlohmann.json;` compiles the library once into a module and @@ -343,6 +431,8 @@ and −2% / 0% (GCC). To get a saving, combine it with - [`JSON_HAS_FILESYSTEM`](../api/macros/json_has_filesystem.md) - control the conversions from and to `std::filesystem::path` +- [`JSON_HAS_STD_FORMAT`](../api/macros/json_has_std_format.md) - control the `std::formatter` specialization +- [`JSON_HAS_RANGES`](../api/macros/json_has_ranges.md) - control the use of `std::ranges` - [`JSON_NO_AUTOMATIC_UDLS`](../api/macros/json_no_automatic_udls.md) - do not include the user-defined string literals automatically - [Modules](../features/modules.md) - C++20 module support diff --git a/docs/mkdocs/scripts/figures/charts.py b/docs/mkdocs/scripts/figures/charts.py index 6f1c4dc3e..d0440977c 100644 --- a/docs/mkdocs/scripts/figures/charts.py +++ b/docs/mkdocs/scripts/figures/charts.py @@ -73,6 +73,8 @@ COMPILE = [ ("JSON_NO_AUTOMATIC_UDLS", "parsing TU", -1, -2, False), ("JSON_HAS_FILESYSTEM=0 + JSON_NO_IO", "model TU", -8, -11, False), ("JSON_HAS_FILESYSTEM=0", "model TU", -5, -9, False), + ("JSON_HAS_STD_FORMAT=0", "model TU", -1, -4, False), + ("JSON_HAS_RANGES=0", "model TU", -4, -1, False), # computed from the ms in "Options without effect on compile times": 1034/1051, 1756/1801, 1055/1051, 1783/1801 ("JSON_NO_IO", "model TU", round((1034 / 1051 - 1) * 100), round((1756 / 1801 - 1) * 100), True), ("JSON_USE_GLOBAL_UDLS=0", "model TU", round((1055 / 1051 - 1) * 100), round((1783 / 1801 - 1) * 100), True), From cbb9e14889f43cd10ffe54870977338381104c6b Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sat, 7 Nov 2026 10:10:21 +0100 Subject: [PATCH 51/54] Fix ci_emscripten: make sweep_allocator::construct() not viable for const types (#6036) * Tests: make sweep_allocator::construct() not viable for const types The newest libc++ (emsdk latest, used by ci_emscripten) probes construct(const T*, const T&) in an unevaluated context while checking whether copies are trivial. For the test allocator, instantiating the noexcept specification then fails to compile, because static_cast cannot drop the const qualifier. Constrain construct() to non-const types, which makes the probe fail in the SFINAE context instead of with a hard error. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01N5NasEsrzHhSNouxSoEwqz Signed-off-by: Niels Lohmann * Fix CI: astyle and clang-tidy for sweep_allocator::construct() astyle 3.4.13 reformats the enable_if default template argument (Check amalgamation). Delete the const overload instead, which makes the libc++ probe for const objects fail in the SFINAE context just the same and needs no . Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01N5NasEsrzHhSNouxSoEwqz Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann Co-authored-by: Claude Sonnet 5.5 --- tests/src/unit-allocator.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/src/unit-allocator.cpp b/tests/src/unit-allocator.cpp index 8fa9034b5..9862e8c5e 100644 --- a/tests/src/unit-allocator.cpp +++ b/tests/src/unit-allocator.cpp @@ -1079,6 +1079,11 @@ struct sweep_allocator ::new (static_cast(p)) U(std::forward(args)...); } + // libc++ probes whether construct() is usable for const objects (when + // checking for trivial copies); this must not be viable then + template + void construct(const U* /*unused*/, const Args& ... /*unused*/) = delete; + template void destroy(U* p) noexcept { From 951271127bacede4e1b3bfc272013a7440ac5b94 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sat, 7 Nov 2026 10:10:50 +0100 Subject: [PATCH 52/54] Binary readers: append strings straight from contiguous input (#6028) * Binary readers: append strings straight from contiguous input get_bytes() read every string into a zero-filled resize() and copied the bytes over it, so each byte was written twice. A string that the input holds in one block (bulk_remaining() at least its length) is now appended from it directly. The claimed length is checked against the input first, so a huge claimed length still allocates nothing it cannot fill; other inputs and binary values keep the chunked loop. from_cbor() and from_msgpack() of twitter.json and citm_catalog.json get 5.5-9.1% faster on an Apple M1 Max and 2.7-8.6% on x86-64 (GCC 15, Clang 21); canada.json, which has few strings, is unchanged. Signed-off-by: Niels Lohmann * Binary readers: keep the tag argument of append_from_block() astyle-proof Signed-off-by: Niels Lohmann * Fix CI: bugprone-signed-char-misuse in append_from_block Convert the (positive) length to its unsigned type before widening it to std::size_t, as clang-tidy flags the int8_t to std::size_t cast. Signed-off-by: Niels Lohmann * Tests: sweep_allocator::construct() accepts pointers to const Recent libc++ (emsdk latest) probes construct(const T*, const T&) in an unevaluated context while checking for trivial copies; the static_cast to void* in the noexcept specification then fails to compile. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01N5NasEsrzHhSNouxSoEwqz (cherry picked from commit 794bf70823c12a2a1cb2a8a95f34dfb6357746a8) Signed-off-by: Niels Lohmann * Tests: guard sweep_allocator::construct() against const types instead of const_cast The const_cast added in the previous commit is rejected by cppcoreguidelines-pro-type-const-cast (clang-tidy). Make construct() not viable for const U instead, which is what libc++ probes for. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01N5NasEsrzHhSNouxSoEwqz (cherry picked from commit de099fa607001b710862fd1bbecef3609c54cbc4) Signed-off-by: Niels Lohmann * Fix CI: astyle and clang-tidy for sweep_allocator::construct() astyle 3.4.13 reformats the enable_if default template argument (Check amalgamation). Delete the const overload instead, which makes the libc++ probe for const objects fail in the SFINAE context just the same and needs no . Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01N5NasEsrzHhSNouxSoEwqz Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann Co-authored-by: Claude Sonnet 5.5 --- .../nlohmann/detail/input/binary_reader.hpp | 38 +++++++++++++++++++ single_include/nlohmann/json.hpp | 38 +++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index 93f707097..81a3ebd42 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -4887,6 +4887,14 @@ class binary_reader const char* context, ContainerType& result) { + // a string that is in the block of input as a whole is appended from + // it, without the zero-filling resize() below + constexpr bool from_block = bulk_scan && std::is_same::value; + if (append_from_block(len, result, std::integral_constant {})) + { + return true; + } + // upper bound on the number of bytes read (and allocated) per chunk constexpr std::size_t chunk_size = 4096; @@ -4923,6 +4931,36 @@ class binary_reader return true; } + /// append the @a len bytes at the current position to @a result, if the + /// input holds them in one block; false (with nothing read) otherwise + template + bool append_from_block(const NumberType len, string_t& result, std::true_type /*block*/) + { + if (len <= 0) + { + return false; + } + // (len > 0, so the unsigned type holds it; casting a signed char + // straight to std::size_t would trip bugprone-signed-char-misuse) + const auto ulen = static_cast::type>(len); + const std::size_t remaining = ia.bulk_remaining(); + if (static_cast(ulen) > static_cast(remaining)) + { + return false; // (the loop reports the premature end) + } + const auto n = static_cast(ulen); + result.append(reinterpret_cast(ia.bulk_data()), n); + ia.bulk_skip(n); + chars_read += n; + return true; + } + + template + static bool append_from_block(const NumberType /*len*/, ContainerType& /*result*/, std::false_type /*block*/) noexcept + { + return false; + } + /*! @param[in] context further context information (for diagnostics) @return whether the last read character is not EOF diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index eedb9c561..6e2ead1b4 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -22903,6 +22903,14 @@ class binary_reader const char* context, ContainerType& result) { + // a string that is in the block of input as a whole is appended from + // it, without the zero-filling resize() below + constexpr bool from_block = bulk_scan && std::is_same::value; + if (append_from_block(len, result, std::integral_constant {})) + { + return true; + } + // upper bound on the number of bytes read (and allocated) per chunk constexpr std::size_t chunk_size = 4096; @@ -22939,6 +22947,36 @@ class binary_reader return true; } + /// append the @a len bytes at the current position to @a result, if the + /// input holds them in one block; false (with nothing read) otherwise + template + bool append_from_block(const NumberType len, string_t& result, std::true_type /*block*/) + { + if (len <= 0) + { + return false; + } + // (len > 0, so the unsigned type holds it; casting a signed char + // straight to std::size_t would trip bugprone-signed-char-misuse) + const auto ulen = static_cast::type>(len); + const std::size_t remaining = ia.bulk_remaining(); + if (static_cast(ulen) > static_cast(remaining)) + { + return false; // (the loop reports the premature end) + } + const auto n = static_cast(ulen); + result.append(reinterpret_cast(ia.bulk_data()), n); + ia.bulk_skip(n); + chars_read += n; + return true; + } + + template + static bool append_from_block(const NumberType /*len*/, ContainerType& /*result*/, std::false_type /*block*/) noexcept + { + return false; + } + /*! @param[in] context further context information (for diagnostics) @return whether the last read character is not EOF From 18138227dc754f0b6293beadd05192de3221b794 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sat, 7 Nov 2026 10:11:09 +0100 Subject: [PATCH 53/54] Scan the buffer of a FILE in bulk with glibc and the Apple C library (#6030) * Scan the buffer of a FILE in bulk with glibc and the Apple C library Parsing from a FILE* still read one character at a time through getc_unlocked(), while a std::istream has its get area scanned in bulk. file_input_adapter now offers the lexer the same kind of window into the buffer of the FILE: it reads the fields that the C library's own getc_unlocked() macro reads (glibc: _IO_read_ptr and _IO_read_end; Apple: _p and _r) and consumes exactly the characters getc_unlocked() would return, so the file is left where it was left before. The part of the current token in a buffer that is about to be refilled is kept for error messages, as for streams. The window needs the file locked for the whole parse, so JSON_DISABLE_UNLOCKED_FILE_IO and MemorySanitizer builds turn it off; uClibc, which defines __GLIBC__ but has other fields, does not get it. parse() from a FILE* gets 2.0-2.6x as fast on an Apple M1 Max and 1.7-2.2x on x86-64 (glibc, GCC 15 and Clang 21), accept() 2.9-4.4x; that is about as fast as parsing from a std::ifstream. The new tests read with FILE buffers of 1 to 70 bytes and unbuffered files, after ungetc(), with SAX positions and with binary strings across the buffer's edges. Signed-off-by: Niels Lohmann * Tests: no setvbuf() buffer below 2 bytes on Windows The Microsoft C runtime rejects a _IOFBF buffer size below 2 as an invalid parameter and ends the process (0xc0000409), which made test-file_input fail on all MSVC and clang-cl jobs. Signed-off-by: Niels Lohmann * Fix CI: astyle formatting in unit-file_input.cpp Signed-off-by: Niels Lohmann * Tests: limit the multi-character ungetc() test to glibc and Apple The Windows C runtime only supports one pushed-back character, which broke the Windows CI jobs. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01N5NasEsrzHhSNouxSoEwqz Signed-off-by: Niels Lohmann * Tests: sweep_allocator::construct() accepts pointers to const Recent libc++ (emsdk latest) probes construct(const T*, const T&) in an unevaluated context while checking for trivial copies; the static_cast to void* in the noexcept specification then fails to compile. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01N5NasEsrzHhSNouxSoEwqz (cherry picked from commit 794bf70823c12a2a1cb2a8a95f34dfb6357746a8) Signed-off-by: Niels Lohmann * Tests: guard sweep_allocator::construct() against const types instead of const_cast The const_cast added in the previous commit is rejected by cppcoreguidelines-pro-type-const-cast (clang-tidy). Make construct() not viable for const U instead, which is what libc++ probes for. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01N5NasEsrzHhSNouxSoEwqz (cherry picked from commit de099fa607001b710862fd1bbecef3609c54cbc4) Signed-off-by: Niels Lohmann * Fix CI: astyle and clang-tidy for sweep_allocator::construct() astyle 3.4.13 reformats the enable_if default template argument (Check amalgamation). Delete the const overload instead, which makes the libc++ probe for const objects fail in the SFINAE context just the same and needs no . Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01N5NasEsrzHhSNouxSoEwqz Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann Co-authored-by: Claude Sonnet 5.5 --- .../macros/json_disable_unlocked_file_io.md | 18 +- docs/mkdocs/docs/features/parsing/index.md | 2 +- docs/mkdocs/docs/features/performance.md | 4 +- .../nlohmann/detail/input/input_adapters.hpp | 179 ++++++++++++++++- include/nlohmann/detail/macro_unscope.hpp | 1 + single_include/nlohmann/json.hpp | 180 +++++++++++++++++- tests/src/unit-file_input.cpp | 133 +++++++++++++ 7 files changed, 506 insertions(+), 11 deletions(-) diff --git a/docs/mkdocs/docs/api/macros/json_disable_unlocked_file_io.md b/docs/mkdocs/docs/api/macros/json_disable_unlocked_file_io.md index 2c266ba2d..031566cfe 100644 --- a/docs/mkdocs/docs/api/macros/json_disable_unlocked_file_io.md +++ b/docs/mkdocs/docs/api/macros/json_disable_unlocked_file_io.md @@ -5,7 +5,9 @@ ``` When defined to `1`, parsing from a `#!cpp FILE*` reads every character with `std::fgetc`, as in version 3.12.0 and -earlier, instead of locking the file once and reading it with the unlocked stdio functions. +earlier, instead of locking the file once and reading it with the unlocked stdio functions. This also turns off the +scan of the `FILE`'s buffer in bulk (see below), which [`JSON_DISABLE_STREAM_BULK_SCAN`](json_disable_stream_bulk_scan.md) +does not affect. It does not change any result: values, exceptions, and the position of the file after parsing are the same either way. Only the speed and the locking of the file differ (see the notes on locking below). @@ -37,9 +39,17 @@ The default value is `0`: the file is locked once and read with the unlocked std | Microsoft (MSVC, clang-cl) | `_lock_file`, `_fgetc_nolock`, `_unlock_file` | | others (e.g., musl, MinGW, newlib) | `std::fgetc`, as before | - This makes parsing from a `FILE*` 3 to 4 times as fast with the Apple C library and 1.1 to 1.3 times as fast with - glibc, whose locking is cheaper. The binary formats ([`from_cbor`](../basic_json/from_cbor.md), - [`from_msgpack`](../basic_json/from_msgpack.md), ...) read from a `FILE*` the same way. + With glibc and the Apple C library, the parser also scans the characters that the `FILE` already holds in its + buffer in bulk, as it scans the get area of a stream (see + [`JSON_DISABLE_STREAM_BULK_SCAN`](json_disable_stream_bulk_scan.md)): it reads the same fields of the `FILE` that + the C library's own `getc_unlocked` reads, and it consumes exactly the characters that `getc_unlocked` would have + returned, so the file is left where it was left before. + + Locking the file once makes parsing from a `FILE*` 3 to 4 times as fast with the Apple C library and 1.1 to 1.3 + times as fast with glibc, whose locking is cheaper. Scanning the buffer in bulk makes `parse()` another 1.7 to 2.6 + times as fast (and `accept()` 2.9 to 4.4 times), about as fast as parsing from a `std::ifstream`. The binary + formats ([`from_cbor`](../basic_json/from_cbor.md), [`from_msgpack`](../basic_json/from_msgpack.md), ...) read + from a `FILE*` the same way. !!! note "Locking" diff --git a/docs/mkdocs/docs/features/parsing/index.md b/docs/mkdocs/docs/features/parsing/index.md index 93a02b501..fa2bec462 100644 --- a/docs/mkdocs/docs/features/parsing/index.md +++ b/docs/mkdocs/docs/features/parsing/index.md @@ -77,7 +77,7 @@ them faster than others: |--------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------| | strings, string views, string literals, contiguous containers, pointer or iterator ranges over contiguous memory | the whole input is scanned in bulk: runs of string characters, numbers, and indentation are recognized several bytes at a time | | `#!cpp std::istream` (`#!cpp std::ifstream`, `#!cpp std::istringstream`, ...) | the characters the stream buffer holds are scanned in bulk like a string; the stream buffer is asked for more when they are used up | -| `#!cpp FILE*` | one character at a time, with the file locked once for the whole parse | +| `#!cpp FILE*` | with glibc and the Apple C library, the characters the `FILE` holds in its buffer are scanned in bulk like a stream's; elsewhere one character at a time; the file is locked once for the whole parse | | other iterator ranges (e.g., `#!cpp std::list`, `#!cpp std::istreambuf_iterator`), wide strings | one character at a time | Parsing a file from a `#!cpp std::ifstream` is therefore only about 10 to 20 % slower than reading the whole file into a diff --git a/docs/mkdocs/docs/features/performance.md b/docs/mkdocs/docs/features/performance.md index 8f4508057..3d0714ca9 100644 --- a/docs/mkdocs/docs/features/performance.md +++ b/docs/mkdocs/docs/features/performance.md @@ -27,8 +27,8 @@ A `#!cpp std::istream` (including `#!cpp std::ifstream`) gets the same bulk scan characters its stream buffer already holds (the *get area*) and asks the stream buffer for more only when they are used up. Nothing is read ahead, so the stream is left exactly where it would be after reading one character at a time, and [`operator>>`](../api/operator_gtgt.md) can still read several values from one stream. A stream buffer without a get -area is read one character at a time. A `#!cpp FILE*` is still read one character at a time, but `file_input_adapter` -locks the file once per parse instead of once per character. See +area is read one character at a time. A `#!cpp FILE*` is locked once per parse instead of once per character, and with +glibc and the Apple C library, the characters in its buffer are scanned in bulk as a stream's get area is. See [input sources and performance](parsing/index.md#input-sources-and-performance) for the full table. According to the description of [#5828](https://github.com/nlohmann/json/pull/5828), parsing from a stream is within about 10 to 20 % of parsing an equivalent `#!cpp std::string`. diff --git a/include/nlohmann/detail/input/input_adapters.hpp b/include/nlohmann/detail/input/input_adapters.hpp index 1d925588f..7b7222f06 100644 --- a/include/nlohmann/detail/input/input_adapters.hpp +++ b/include/nlohmann/detail/input/input_adapters.hpp @@ -58,6 +58,19 @@ #define JSON_UNLOCKED_FILE_IO 0 #endif #undef JSON_UNLOCKED_FILE_IO_MSAN + + // Whether file_input_adapter scans the buffer of the FILE directly, as + // getc_unlocked() reads it: the fields it uses are those that the C + // library's own getc_unlocked() macro uses (glibc: _IO_read_ptr and + // _IO_read_end; Apple: _p and _r). It needs the file locked for the whole + // parse, as JSON_UNLOCKED_FILE_IO == 1 has it (so JSON_DISABLE_UNLOCKED_FILE_IO + // and MemorySanitizer builds turn it off). + // (uClibc defines __GLIBC__ as well, but its FILE has other fields) + #if JSON_UNLOCKED_FILE_IO == 1 && ((defined(__GLIBC__) && !defined(__UCLIBC__)) || defined(__APPLE__)) + #define JSON_FILE_WINDOW 1 + #else + #define JSON_FILE_WINDOW 0 + #endif #endif // JSON_NO_IO NLOHMANN_JSON_NAMESPACE_BEGIN @@ -179,8 +192,12 @@ class token_carry_buffer char* m_capacity_end = nullptr; }; /*! -Input adapter for stdio file access. This adapter read only 1 byte and do not use any - buffer. This adapter is a very low level adapter. +Input adapter for stdio file access. The file is locked for the lifetime of the +adapter and read with the unlocked stdio functions (see +JSON_DISABLE_UNLOCKED_FILE_IO). With glibc and the Apple C library, the +characters in the buffer of the FILE are also scanned in bulk, as the get area +of a stream is (see input_stream_adapter): exactly the characters getc_unlocked() +would return are consumed, so the file is left where it was left before. */ class file_input_adapter { @@ -199,6 +216,12 @@ class file_input_adapter file_input_adapter(const file_input_adapter&) = delete; file_input_adapter(file_input_adapter&& rhs) noexcept : m_file(rhs.m_file) +#if JSON_FILE_WINDOW + , token_carry(std::move(rhs.token_carry)) + , token_segment(rhs.token_segment) + , current_in_buffer(rhs.current_in_buffer) + , tracking_token(rhs.tracking_token) +#endif { // the lock is owned by the thread, not the adapter: hand it over rhs.m_file = nullptr; @@ -210,6 +233,20 @@ class file_input_adapter unlock(); } +#if JSON_FILE_WINDOW + std::char_traits::int_type get_character() + { + // what getc_unlocked() does, while the buffer has characters + const char* const next = buffer_ptr(); + if (JSON_HEDLEY_LIKELY(next != buffer_end())) + { + current_in_buffer = true; + buffer_bump(1); + return std::char_traits::to_int_type(*next); + } + return get_character_slow(); + } +#else std::char_traits::int_type get_character() noexcept { // the file is locked for the lifetime of the adapter, so the @@ -222,6 +259,7 @@ class file_input_adapter return std::fgetc(m_file); #endif } +#endif // returns the number of characters successfully read template @@ -230,7 +268,134 @@ class file_input_adapter return fread(dest, 1, sizeof(T) * count, m_file); } +#if JSON_FILE_WINDOW + // The lexer may scan the buffer of the FILE directly, as the window of + // input_stream_adapter: the buffer ends wherever the C library's buffer + // ends, and is refilled by the next get_character(). + static constexpr bool supports_bulk_scan = true; + static constexpr bool bulk_window = true; + + // Pointer to the next unread character; only valid when bulk_remaining() > 0. + const char* bulk_data() const + { + return buffer_ptr(); + } + + // Number of unread characters in the buffer. + std::size_t bulk_remaining() const + { + if (!current_in_buffer) + { + return 0; + } + return static_cast(buffer_end() - buffer_ptr()); + } + + // Consume @a n characters previously inspected via bulk_data(). + void bulk_skip(std::size_t n) + { + JSON_ASSERT(n <= bulk_remaining()); + buffer_bump(n); + } + + // The adapter keeps the current token for the lexer's error messages, as + // input_stream_adapter does. + static constexpr bool supports_token_tracking = true; + + void begin_token() noexcept + { + tracking_token = true; + token_carry.clear(); + token_segment = buffer_ptr(); + } + + void mark_token_start(std::char_traits::int_type c) + { + tracking_token = true; + token_carry.clear(); + if (current_in_buffer) + { + token_segment = buffer_ptr() - 1; + } + else + { + token_segment = nullptr; + token_carry.push_back(std::char_traits::to_char_type(c)); + } + } + + void copy_token(std::vector& out, bool drop_last) const + { + token_carry.copy_to(out); + if (token_segment != nullptr) + { + out.insert(out.end(), token_segment, buffer_ptr()); + } + if (drop_last && !out.empty()) + { + out.pop_back(); + } + } +#endif + private: +#if JSON_FILE_WINDOW + // the buffer of the FILE: [buffer_ptr(), buffer_end()) are the characters + // getc_unlocked() would return next +#if defined(__GLIBC__) + const char* buffer_ptr() const noexcept + { + return m_file->_IO_read_ptr; + } + const char* buffer_end() const noexcept + { + return m_file->_IO_read_end; + } + void buffer_bump(std::size_t n) noexcept + { + m_file->_IO_read_ptr += n; + } +#else + const char* buffer_ptr() const noexcept + { + return reinterpret_cast(m_file->_p); + } + const char* buffer_end() const noexcept + { + return m_file->_r > 0 ? reinterpret_cast(m_file->_p) + m_file->_r : reinterpret_cast(m_file->_p); + } + void buffer_bump(std::size_t n) noexcept + { + m_file->_p += n; + m_file->_r -= static_cast(n); + } +#endif + + // the buffer is exhausted: getc_unlocked() refills it + JSON_HEDLEY_NEVER_INLINE std::char_traits::int_type get_character_slow() + { + if (token_segment != nullptr) + { + token_carry.append(token_segment, buffer_end()); + token_segment = nullptr; + } + const auto res = getc_unlocked(m_file); + if (res == EOF) + { + current_in_buffer = false; + return std::char_traits::eof(); + } + // the character was taken from the refilled buffer, at buffer_ptr() - 1 + current_in_buffer = true; + if (tracking_token) + { + token_segment = buffer_ptr() - 1; + } + return res; + } + +#endif + // Lock the file once for the whole parse instead of once per character. // The lock is recursive, so fread() and the user's own stdio calls on the // same thread keep working. @@ -259,6 +424,16 @@ class file_input_adapter /// the file pointer to read from std::FILE* m_file; +#if JSON_FILE_WINDOW + /// the part of the current token that was in earlier buffers + token_carry_buffer token_carry{}; // NOLINT(readability-redundant-member-init) + /// start of the current token's part in the buffer, or nullptr + const char* token_segment = nullptr; + /// whether the character last returned by get_character() is in the buffer + bool current_in_buffer = false; + /// whether a token is tracked at all; the binary readers never start one + bool tracking_token = false; +#endif }; /*! diff --git a/include/nlohmann/detail/macro_unscope.hpp b/include/nlohmann/detail/macro_unscope.hpp index d3cdb0d3f..b61db5dc6 100644 --- a/include/nlohmann/detail/macro_unscope.hpp +++ b/include/nlohmann/detail/macro_unscope.hpp @@ -22,6 +22,7 @@ #undef JSON_DISABLE_ENUM_SERIALIZATION #undef JSON_DISABLE_TUPLE_REFERENCE_CONVERSION #undef JSON_UNLOCKED_FILE_IO +#undef JSON_FILE_WINDOW #undef JSON_DTOA_SSE2 #undef JSON_DTOA_NEON #undef JSON_LITTLE_ENDIAN_TARGET diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 6e2ead1b4..f91671e82 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -9167,6 +9167,19 @@ NLOHMANN_JSON_NAMESPACE_END #define JSON_UNLOCKED_FILE_IO 0 #endif #undef JSON_UNLOCKED_FILE_IO_MSAN + + // Whether file_input_adapter scans the buffer of the FILE directly, as + // getc_unlocked() reads it: the fields it uses are those that the C + // library's own getc_unlocked() macro uses (glibc: _IO_read_ptr and + // _IO_read_end; Apple: _p and _r). It needs the file locked for the whole + // parse, as JSON_UNLOCKED_FILE_IO == 1 has it (so JSON_DISABLE_UNLOCKED_FILE_IO + // and MemorySanitizer builds turn it off). + // (uClibc defines __GLIBC__ as well, but its FILE has other fields) + #if JSON_UNLOCKED_FILE_IO == 1 && ((defined(__GLIBC__) && !defined(__UCLIBC__)) || defined(__APPLE__)) + #define JSON_FILE_WINDOW 1 + #else + #define JSON_FILE_WINDOW 0 + #endif #endif // JSON_NO_IO NLOHMANN_JSON_NAMESPACE_BEGIN @@ -9288,8 +9301,12 @@ class token_carry_buffer char* m_capacity_end = nullptr; }; /*! -Input adapter for stdio file access. This adapter read only 1 byte and do not use any - buffer. This adapter is a very low level adapter. +Input adapter for stdio file access. The file is locked for the lifetime of the +adapter and read with the unlocked stdio functions (see +JSON_DISABLE_UNLOCKED_FILE_IO). With glibc and the Apple C library, the +characters in the buffer of the FILE are also scanned in bulk, as the get area +of a stream is (see input_stream_adapter): exactly the characters getc_unlocked() +would return are consumed, so the file is left where it was left before. */ class file_input_adapter { @@ -9308,6 +9325,12 @@ class file_input_adapter file_input_adapter(const file_input_adapter&) = delete; file_input_adapter(file_input_adapter&& rhs) noexcept : m_file(rhs.m_file) +#if JSON_FILE_WINDOW + , token_carry(std::move(rhs.token_carry)) + , token_segment(rhs.token_segment) + , current_in_buffer(rhs.current_in_buffer) + , tracking_token(rhs.tracking_token) +#endif { // the lock is owned by the thread, not the adapter: hand it over rhs.m_file = nullptr; @@ -9319,6 +9342,20 @@ class file_input_adapter unlock(); } +#if JSON_FILE_WINDOW + std::char_traits::int_type get_character() + { + // what getc_unlocked() does, while the buffer has characters + const char* const next = buffer_ptr(); + if (JSON_HEDLEY_LIKELY(next != buffer_end())) + { + current_in_buffer = true; + buffer_bump(1); + return std::char_traits::to_int_type(*next); + } + return get_character_slow(); + } +#else std::char_traits::int_type get_character() noexcept { // the file is locked for the lifetime of the adapter, so the @@ -9331,6 +9368,7 @@ class file_input_adapter return std::fgetc(m_file); #endif } +#endif // returns the number of characters successfully read template @@ -9339,7 +9377,134 @@ class file_input_adapter return fread(dest, 1, sizeof(T) * count, m_file); } +#if JSON_FILE_WINDOW + // The lexer may scan the buffer of the FILE directly, as the window of + // input_stream_adapter: the buffer ends wherever the C library's buffer + // ends, and is refilled by the next get_character(). + static constexpr bool supports_bulk_scan = true; + static constexpr bool bulk_window = true; + + // Pointer to the next unread character; only valid when bulk_remaining() > 0. + const char* bulk_data() const + { + return buffer_ptr(); + } + + // Number of unread characters in the buffer. + std::size_t bulk_remaining() const + { + if (!current_in_buffer) + { + return 0; + } + return static_cast(buffer_end() - buffer_ptr()); + } + + // Consume @a n characters previously inspected via bulk_data(). + void bulk_skip(std::size_t n) + { + JSON_ASSERT(n <= bulk_remaining()); + buffer_bump(n); + } + + // The adapter keeps the current token for the lexer's error messages, as + // input_stream_adapter does. + static constexpr bool supports_token_tracking = true; + + void begin_token() noexcept + { + tracking_token = true; + token_carry.clear(); + token_segment = buffer_ptr(); + } + + void mark_token_start(std::char_traits::int_type c) + { + tracking_token = true; + token_carry.clear(); + if (current_in_buffer) + { + token_segment = buffer_ptr() - 1; + } + else + { + token_segment = nullptr; + token_carry.push_back(std::char_traits::to_char_type(c)); + } + } + + void copy_token(std::vector& out, bool drop_last) const + { + token_carry.copy_to(out); + if (token_segment != nullptr) + { + out.insert(out.end(), token_segment, buffer_ptr()); + } + if (drop_last && !out.empty()) + { + out.pop_back(); + } + } +#endif + private: +#if JSON_FILE_WINDOW + // the buffer of the FILE: [buffer_ptr(), buffer_end()) are the characters + // getc_unlocked() would return next +#if defined(__GLIBC__) + const char* buffer_ptr() const noexcept + { + return m_file->_IO_read_ptr; + } + const char* buffer_end() const noexcept + { + return m_file->_IO_read_end; + } + void buffer_bump(std::size_t n) noexcept + { + m_file->_IO_read_ptr += n; + } +#else + const char* buffer_ptr() const noexcept + { + return reinterpret_cast(m_file->_p); + } + const char* buffer_end() const noexcept + { + return m_file->_r > 0 ? reinterpret_cast(m_file->_p) + m_file->_r : reinterpret_cast(m_file->_p); + } + void buffer_bump(std::size_t n) noexcept + { + m_file->_p += n; + m_file->_r -= static_cast(n); + } +#endif + + // the buffer is exhausted: getc_unlocked() refills it + JSON_HEDLEY_NEVER_INLINE std::char_traits::int_type get_character_slow() + { + if (token_segment != nullptr) + { + token_carry.append(token_segment, buffer_end()); + token_segment = nullptr; + } + const auto res = getc_unlocked(m_file); + if (res == EOF) + { + current_in_buffer = false; + return std::char_traits::eof(); + } + // the character was taken from the refilled buffer, at buffer_ptr() - 1 + current_in_buffer = true; + if (tracking_token) + { + token_segment = buffer_ptr() - 1; + } + return res; + } + +#endif + // Lock the file once for the whole parse instead of once per character. // The lock is recursive, so fread() and the user's own stdio calls on the // same thread keep working. @@ -9368,6 +9533,16 @@ class file_input_adapter /// the file pointer to read from std::FILE* m_file; +#if JSON_FILE_WINDOW + /// the part of the current token that was in earlier buffers + token_carry_buffer token_carry{}; // NOLINT(readability-redundant-member-init) + /// start of the current token's part in the buffer, or nullptr + const char* token_segment = nullptr; + /// whether the character last returned by get_character() is in the buffer + bool current_in_buffer = false; + /// whether a token is tracked at all; the binary readers never start one + bool tracking_token = false; +#endif }; /*! @@ -46386,6 +46561,7 @@ struct formatter // NOLINT(cert-dcl58-c #undef JSON_DISABLE_ENUM_SERIALIZATION #undef JSON_DISABLE_TUPLE_REFERENCE_CONVERSION #undef JSON_UNLOCKED_FILE_IO +#undef JSON_FILE_WINDOW #undef JSON_DTOA_SSE2 #undef JSON_DTOA_NEON #undef JSON_LITTLE_ENDIAN_TARGET diff --git a/tests/src/unit-file_input.cpp b/tests/src/unit-file_input.cpp index e88b6f4d0..f74f23267 100644 --- a/tests/src/unit-file_input.cpp +++ b/tests/src/unit-file_input.cpp @@ -49,6 +49,14 @@ file_ptr open_with(const char* name, const std::string& content) return {std::fopen(name, "rb"), &std::fclose}; } +// the smallest size setvbuf() accepts for _IOFBF: the Microsoft C runtime +// rejects sizes below 2 as an invalid parameter, which ends the process +#if defined(_WIN32) + constexpr int min_full_buffer = 2; +#else + constexpr int min_full_buffer = 1; +#endif + // The stdio lock is recursive, so whether parsing released it can only be // seen from another thread: ftrylockfile() fails there while this thread // holds the lock. ftrylockfile() is POSIX; elsewhere, the check is skipped. @@ -225,6 +233,131 @@ TEST_CASE("parsing from a FILE*") check_unlocked(f.get()); } +#if !defined(JSON_NOEXCEPTION) + SECTION("tokens across the edges of the file's buffer") + { + // the adapter reads the buffer of the FILE directly where it can (see + // JSON_FILE_WINDOW): with buffers of 1 to 70 bytes and none, every + // token crosses an edge somewhere; the results, the error messages + // and their positions must be those of the string + const std::vector documents = + { + input, + R"({"long": "a string that is longer than the buffer of the file, with \"escapes\" and \u00e4", "n": [123456789012345678901, -0.000123e-45, 1.5E+300, 0, -0]})", + "[\n \"indented\",\n \"caf\xc3\xa9 \xe2\x82\xac \xf0\x9f\x98\x80\",\n {\n \"key\": 1\n }\n]", + R"({"a": [1, 2, 3], "b": "unterminated)", + R"([1, 2, 3, 1e999])", + R"({"a": "invalid escape \x"})", + "[\"invalid UTF-8 \xc3\x28\"]", + "[1, 2, 3] trailing", + R"([truex])", + }; + for (const auto& document : documents) + { + std::string expected; + try + { + expected = json::parse(document).dump(); + } + catch (const json::exception& e) + { + expected = e.what(); + } + const bool expected_accept = json::accept(document); + for (int size = 0; size <= 70; ++size) + { + if (size != 0 && size < min_full_buffer) + { + continue; + } + CAPTURE(document) + CAPTURE(size) + std::vector buffer(static_cast(size) + 1); + for (int pass = 0; pass < 2; ++pass) + { + const file_ptr f = open_with(name, document); + REQUIRE(f != nullptr); + // size 0: unbuffered + REQUIRE(std::setvbuf(f.get(), size == 0 ? nullptr : buffer.data(), size == 0 ? _IONBF : _IOFBF, static_cast(size)) == 0); + if (pass == 0) + { + std::string actual; + try + { + actual = json::parse(f.get()).dump(); + } + catch (const json::exception& e) + { + actual = e.what(); + } + CHECK(actual == expected); + } + else + { + CHECK(json::accept(f.get()) == expected_accept); + } + check_unlocked(f.get()); + } + } + } + } +#endif + + // the C standard guarantees only one pushed-back character (and the + // Windows C runtime fails further calls to ungetc()), so this is limited + // to the C libraries that keep them in a separate buffer +#if !defined(JSON_NOEXCEPTION) && ((defined(__GLIBC__) && !defined(__UCLIBC__)) || defined(__APPLE__)) + SECTION("characters pushed back with ungetc() before parsing") + { + const file_ptr f = open_with(name, "2, 3]"); + REQUIRE(f != nullptr); + REQUIRE(std::ungetc(',', f.get()) == ','); + REQUIRE(std::ungetc('1', f.get()) == '1'); + REQUIRE(std::ungetc('[', f.get()) == '['); + CHECK(json::parse(f.get()) == json({1, 2, 3})); + } +#endif + + SECTION("SAX events across the edges of the file's buffer") + { + const std::string document = R"({"key": "a value that is longer than the buffer", "n": [1, -2.5e3, true, null]})"; + std::vector expected; + { + const file_ptr f = open_with(name, document); + REQUIRE(f != nullptr); + position_reading_sax sax(f.get()); + CHECK(json::sax_parse(f.get(), &sax)); + expected = sax.positions(); + } + for (int size = min_full_buffer; size <= 20; ++size) + { + CAPTURE(size) + std::vector buffer(static_cast(size)); + const file_ptr f = open_with(name, document); + REQUIRE(f != nullptr); + REQUIRE(std::setvbuf(f.get(), buffer.data(), _IOFBF, buffer.size()) == 0); + position_reading_sax sax(f.get()); + CHECK(json::sax_parse(f.get(), &sax)); + // the events happen at the same positions of the file + CHECK(sax.positions() == expected); + } + } + + SECTION("binary strings across the edges of the file's buffer") + { + const json value = {{"a string that is longer than the buffer of the file", "and a value that is longer, too"}, {"n", {1, 2}}}; + const std::vector cbor = json::to_cbor(value); + for (int size = min_full_buffer; size <= 20; ++size) + { + CAPTURE(size) + std::vector buffer(static_cast(size)); + const file_ptr f = open_with(name, std::string(cbor.begin(), cbor.end())); + REQUIRE(f != nullptr); + REQUIRE(std::setvbuf(f.get(), buffer.data(), _IOFBF, buffer.size()) == 0); + CHECK(json::from_cbor(f.get()) == value); + } + } + SECTION("binary formats read from a FILE*") { const json j = {{"string", std::string(300, 'x')}, {"array", {1, 2, 3}}, {"number", 1.5}}; From 7fc8312c64fc6a507387b228fdc1e5ae122432d0 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sat, 7 Nov 2026 14:27:52 +0100 Subject: [PATCH 54/54] Fix ci_test_clang_sanitizer: build the CBOR test string from bytes (#6037) --- tests/src/unit-file_input.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/src/unit-file_input.cpp b/tests/src/unit-file_input.cpp index f74f23267..ce190dfed 100644 --- a/tests/src/unit-file_input.cpp +++ b/tests/src/unit-file_input.cpp @@ -351,7 +351,7 @@ TEST_CASE("parsing from a FILE*") { CAPTURE(size) std::vector buffer(static_cast(size)); - const file_ptr f = open_with(name, std::string(cbor.begin(), cbor.end())); + const file_ptr f = open_with(name, std::string(reinterpret_cast(cbor.data()), cbor.size())); REQUIRE(f != nullptr); REQUIRE(std::setvbuf(f.get(), buffer.data(), _IOFBF, buffer.size()) == 0); CHECK(json::from_cbor(f.get()) == value);