From 3e2de322265763088f4aff5c50f3f07857e70e44 Mon Sep 17 00:00:00 2001 From: Wu Shuwen Date: Thu, 10 Sep 2026 14:27:05 +0800 Subject: [PATCH 01/18] Remove unused iomanip include (#5516) Signed-off-by: dajiaohuang --- include/nlohmann/detail/output/serializer.hpp | 1 - single_include/nlohmann/json.hpp | 1 - 2 files changed, 2 deletions(-) diff --git a/include/nlohmann/detail/output/serializer.hpp b/include/nlohmann/detail/output/serializer.hpp index 0b608f8e2..857fc2445 100644 --- a/include/nlohmann/detail/output/serializer.hpp +++ b/include/nlohmann/detail/output/serializer.hpp @@ -18,7 +18,6 @@ #include // snprintf #include // numeric_limits #include // string, char_traits -#include // setfill, setw #include // is_same #include // move diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index d3c8293a7..63d22b1f4 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -19122,7 +19122,6 @@ NLOHMANN_JSON_NAMESPACE_END #include // snprintf #include // numeric_limits #include // string, char_traits -#include // setfill, setw #include // is_same #include // move From d69fb8654d837b1c19ab130c136757bdecbc447b Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Thu, 10 Sep 2026 17:15:34 +0200 Subject: [PATCH 02/18] Return the parsed value by move from from_cbor() and friends (#5501) The binary entry points end with return res ? result : basic_json(value_t::discarded); The condition operator's second operand is an lvalue, so this is not a case where the return value can be elided or implicitly moved from: every successful from_cbor(), from_msgpack(), from_ubjson(), from_bjdata() and from_bson() call deep-copies the value it just parsed, and then destroys the original. The copy is not cheap, and it is not incidental: basic_json's copy constructor walks the whole value. Parsing a 2 MB CBOR document with 60,000 objects, median of 25 runs, clang 17 -O3: from_cbor 26.99 ms -> 14.65 ms from_msgpack 26.82 ms -> 14.82 ms Moving instead of copying is the entire change; the parsed value is not used again after the return expression is evaluated. There is a second reason to prefer the move. The copy constructor recurses once per nesting level, so the copy is also a stack-overflow path on the return side, on a value the reader has already accepted. That is currently masked because the readers themselves recurse and overflow first (#5104), but it has to be fixed for making them iterative to have any effect. Signed-off-by: Niels Lohmann --- include/nlohmann/json.hpp | 98 +++++++++++++++++++++++--------- single_include/nlohmann/json.hpp | 98 +++++++++++++++++++++++--------- 2 files changed, 140 insertions(+), 56 deletions(-) diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index fea75d57a..9bbd98f15 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -4501,8 +4501,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; auto ia = detail::input_adapter(std::forward(i)); detail::json_sax_dom_parser sdp(result, allow_exceptions); - const bool res = binary_reader(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } /// @brief create a JSON value from an input in CBOR format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -4518,8 +4521,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; auto ia = detail::input_adapter(std::move(first), std::move(last)); detail::json_sax_dom_parser sdp(result, allow_exceptions); - const bool res = binary_reader(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } template @@ -4544,8 +4550,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec auto ia = i.get(); detail::json_sax_dom_parser sdp(result, allow_exceptions); // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) - const bool res = binary_reader(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } /// @brief create a JSON value from an input in MessagePack format @@ -4559,8 +4568,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; auto ia = detail::input_adapter(std::forward(i)); detail::json_sax_dom_parser sdp(result, allow_exceptions); - const bool res = binary_reader(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } /// @brief create a JSON value from an input in MessagePack format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -4575,8 +4587,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; auto ia = detail::input_adapter(std::move(first), std::move(last)); detail::json_sax_dom_parser sdp(result, allow_exceptions); - const bool res = binary_reader(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } template @@ -4599,8 +4614,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec auto ia = i.get(); detail::json_sax_dom_parser sdp(result, allow_exceptions); // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) - const bool res = binary_reader(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } /// @brief create a JSON value from an input in UBJSON format @@ -4614,8 +4632,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; auto ia = detail::input_adapter(std::forward(i)); detail::json_sax_dom_parser sdp(result, allow_exceptions); - const bool res = binary_reader(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } /// @brief create a JSON value from an input in UBJSON format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -4630,8 +4651,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; auto ia = detail::input_adapter(std::move(first), std::move(last)); detail::json_sax_dom_parser sdp(result, allow_exceptions); - const bool res = binary_reader(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } template @@ -4654,8 +4678,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec auto ia = i.get(); detail::json_sax_dom_parser sdp(result, allow_exceptions); // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) - const bool res = binary_reader(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } /// @brief create a JSON value from an input in BJData format @@ -4669,8 +4696,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; auto ia = detail::input_adapter(std::forward(i)); detail::json_sax_dom_parser sdp(result, allow_exceptions); - const bool res = binary_reader(std::move(ia), input_format_t::bjdata).sax_parse(input_format_t::bjdata, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::bjdata).sax_parse(input_format_t::bjdata, &sdp, strict)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } /// @brief create a JSON value from an input in BJData format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -4685,8 +4715,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; auto ia = detail::input_adapter(std::move(first), std::move(last)); detail::json_sax_dom_parser sdp(result, allow_exceptions); - const bool res = binary_reader(std::move(ia), input_format_t::bjdata).sax_parse(input_format_t::bjdata, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::bjdata).sax_parse(input_format_t::bjdata, &sdp, strict)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } /// @brief create a JSON value from an input in BSON format @@ -4700,8 +4733,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; auto ia = detail::input_adapter(std::forward(i)); detail::json_sax_dom_parser sdp(result, allow_exceptions); - const bool res = binary_reader(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } /// @brief create a JSON value from an input in BSON format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -4716,8 +4752,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; auto ia = detail::input_adapter(std::move(first), std::move(last)); detail::json_sax_dom_parser sdp(result, allow_exceptions); - const bool res = binary_reader(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } template @@ -4740,8 +4779,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec auto ia = i.get(); detail::json_sax_dom_parser sdp(result, allow_exceptions); // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) - const bool res = binary_reader(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } /// @} diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 63d22b1f4..1d1f290bc 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -26080,8 +26080,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; auto ia = detail::input_adapter(std::forward(i)); detail::json_sax_dom_parser sdp(result, allow_exceptions); - const bool res = binary_reader(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } /// @brief create a JSON value from an input in CBOR format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -26097,8 +26100,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; auto ia = detail::input_adapter(std::move(first), std::move(last)); detail::json_sax_dom_parser sdp(result, allow_exceptions); - const bool res = binary_reader(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } template @@ -26123,8 +26129,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec auto ia = i.get(); detail::json_sax_dom_parser sdp(result, allow_exceptions); // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) - const bool res = binary_reader(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } /// @brief create a JSON value from an input in MessagePack format @@ -26138,8 +26147,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; auto ia = detail::input_adapter(std::forward(i)); detail::json_sax_dom_parser sdp(result, allow_exceptions); - const bool res = binary_reader(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } /// @brief create a JSON value from an input in MessagePack format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -26154,8 +26166,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; auto ia = detail::input_adapter(std::move(first), std::move(last)); detail::json_sax_dom_parser sdp(result, allow_exceptions); - const bool res = binary_reader(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } template @@ -26178,8 +26193,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec auto ia = i.get(); detail::json_sax_dom_parser sdp(result, allow_exceptions); // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) - const bool res = binary_reader(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } /// @brief create a JSON value from an input in UBJSON format @@ -26193,8 +26211,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; auto ia = detail::input_adapter(std::forward(i)); detail::json_sax_dom_parser sdp(result, allow_exceptions); - const bool res = binary_reader(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } /// @brief create a JSON value from an input in UBJSON format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -26209,8 +26230,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; auto ia = detail::input_adapter(std::move(first), std::move(last)); detail::json_sax_dom_parser sdp(result, allow_exceptions); - const bool res = binary_reader(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } template @@ -26233,8 +26257,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec auto ia = i.get(); detail::json_sax_dom_parser sdp(result, allow_exceptions); // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) - const bool res = binary_reader(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } /// @brief create a JSON value from an input in BJData format @@ -26248,8 +26275,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; auto ia = detail::input_adapter(std::forward(i)); detail::json_sax_dom_parser sdp(result, allow_exceptions); - const bool res = binary_reader(std::move(ia), input_format_t::bjdata).sax_parse(input_format_t::bjdata, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::bjdata).sax_parse(input_format_t::bjdata, &sdp, strict)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } /// @brief create a JSON value from an input in BJData format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -26264,8 +26294,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; auto ia = detail::input_adapter(std::move(first), std::move(last)); detail::json_sax_dom_parser sdp(result, allow_exceptions); - const bool res = binary_reader(std::move(ia), input_format_t::bjdata).sax_parse(input_format_t::bjdata, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::bjdata).sax_parse(input_format_t::bjdata, &sdp, strict)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } /// @brief create a JSON value from an input in BSON format @@ -26279,8 +26312,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; auto ia = detail::input_adapter(std::forward(i)); detail::json_sax_dom_parser sdp(result, allow_exceptions); - const bool res = binary_reader(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } /// @brief create a JSON value from an input in BSON format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -26295,8 +26331,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; auto ia = detail::input_adapter(std::move(first), std::move(last)); detail::json_sax_dom_parser sdp(result, allow_exceptions); - const bool res = binary_reader(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } template @@ -26319,8 +26358,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec auto ia = i.get(); detail::json_sax_dom_parser sdp(result, allow_exceptions); // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) - const bool res = binary_reader(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + if (!binary_reader(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict)) // cppcheck-suppress[accessMoved] + { + result = value_t::discarded; + } + return result; } /// @} From 4a93aa4e2fa4bda7194795e4b810a25f1ff6f4fd Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Thu, 10 Sep 2026 17:20:48 +0200 Subject: [PATCH 03/18] Speed up parsing of contiguous input (numbers, strings, UTF-8) (#5283) * Speed up number parsing in the lexer (fast paths from the fast_float/simdjson world) The number scanner converted its already-validated digit buffer with std::strtoull/std::strtoll/std::strtod. Those pull in locale and errno machinery and dominate number-heavy parsing (strtod runs at ~6 M/s). Replace them with dedicated parsers over the validated buffer: - parse_integer_unsigned / parse_integer_signed: accumulate digits with overflow detection, falling back to the float path on overflow exactly as the strtoull/strtoll round-trip check did. Overflow behavior is unchanged for narrower or wider custom number types. - parse_float_fast: Clinger's exact fast path for `double` (<=19 significant digits, |exp10| <= 22, significand < 2^53), where significand * 10^exp is exact under IEEE round-to-nearest. This is the same fast path used by fast_float/simdjson. It is bit-identical to strtod on this subset and declines (falling back to strtod) otherwise. Only `double` uses it; float and long double keep std::strtof/std::strtold via a templated overload. Measured on representative data (g++ 13, -O3): - integers: DOM parse +11%, SAX +25-34% - floats: DOM parse +37%, SAX +70% (clang: float DOM ~1.9x) No dependencies added; header-only and C++11-clean. Existing parser, lexer, conversion and deserialization unit tests pass unchanged; a 3M-value random-double fuzz matches strtod bit-for-bit. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01AXcDtEma2PjxgmPS9cQGzA Signed-off-by: Niels Lohmann * Add SWAR bulk string scanning for contiguous input (simdjson-style) scan_string() read the input one character at a time through the input adapter and classified every byte with a large switch. For contiguous byte buffers we can instead scan 8 bytes at a time with a SWAR word test that finds the first byte needing individual handling (the closing quote, an escape, a control character, or a non-ASCII UTF-8 byte) and bulk-append the ordinary run in one go. - input adapters expose supports_bulk_scan / bulk_data / bulk_remaining / bulk_skip for provably-contiguous, same-type, 1-byte iterator ranges (raw pointers in every standard; std::string/std::vector/std::array and friends additionally in C++20 via std::contiguous_iterator). - the lexer gains a bulk_scan capability (gated on lazy_token_string so bypassing the per-character capture cannot lose error diagnostics) and a scan_string_bulk() fast path; streaming/wide/user adapters are unchanged and keep the byte-at-a-time scanner. The run contains no newline (all bytes < 0x20 are treated as special), so position bookkeeping stays exact, and error tokens are still reconstructed lazily from the consumed byte range. The SWAR special-byte test is pure uint64_t arithmetic - no intrinsics, no runtime dispatch, C++11-clean. Measured on representative data, pointer input, g++ 13 -O3 (string values discarded by accept() see the largest gains): long ASCII strings: DOM +4.5x, SAX +14x, accept +17x (to ~2 GB/s) short strings: DOM +15%, SAX +62%, accept +85% escape-heavy: DOM +31%, SAX +26%, accept +28% Same-input parity verified: 200k randomized documents (escapes, multibyte UTF-8, surrogate pairs) accept/parse identically via the contiguous SWAR path and the streaming byte path; unit lexer/parser/diagnostic-position/ deserialization/conversions suites pass unchanged. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01AXcDtEma2PjxgmPS9cQGzA Signed-off-by: Niels Lohmann * Validate UTF-8 in the bulk string scanner (portable ~2-3x on non-ASCII text) The SWAR bulk string path stopped at the first non-ASCII byte and handed every multibyte character to the byte-at-a-time scanner, whose per-byte get()/next_byte_in_range()/add() machinery runs at roughly half the speed of validating straight from the buffer. As a result, dense non-ASCII text (CJK, emoji, accented Latin) parsed ~10-15x slower than ASCII. Fold well-formed UTF-8 into the bulk run: scan_string_bulk() now, on a non-ASCII lead byte, validates one sequence with validate_one_utf8() - which mirrors scan_string()'s per-byte switch ranges exactly (rejecting overlong forms, surrogates, and out-of-range code points) - and appends it in place, continuing until the closing quote, an escape, a control byte, or an ill-formed sequence. All error handling still defers to the byte path, so error messages and positions are byte-for-byte unchanged. Because only well-formed content is fast-pathed and every rejection falls through to the existing scanner, behavior is identical; the win is purely throughput. Measured on pointer input (accept, string values discarded): content g++ 13 clang 18 dense CJK 277 -> 648 ~605 MB/s (~2.3x) dense emoji 299 -> 857 ~702 MB/s (~2.6-2.9x) mixed 90% ASCII 246 -> 331 ~334 MB/s (~1.35x) pure ASCII unchanged (~3.2 / 4.1 GB/s) Verified: 2,000,000 randomized documents built from arbitrary bytes (overlong, surrogate, truncated, out-of-range sequences) accept/reject and parse identically via the contiguous path and the streaming byte path; lexer/parser/diagnostic-position/deserialization/conversions suites pass unchanged. Pure C++11, no intrinsics. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01AXcDtEma2PjxgmPS9cQGzA Signed-off-by: Niels Lohmann * Route contiguous byte containers through the pointer adapter (fast paths in C++11) json::parse(std::string) - the most common entry point - did not benefit from the contiguous fast paths (bulk string scanning, UTF-8 bulk validation, memcpy for binary formats) in C++11..17: std::string::iterator is a library wrapper, not a raw pointer, and pre-C++20 there is no portable way to prove it contiguous, so supports_bulk_scan was false. Only raw pointers, string literals, and C-arrays (and, in C++20, anything modelling std::contiguous_iterator) took the fast path. Detect contiguous single-byte containers (std::string, std::vector, std::vector, std::string_view, ...) via is_contiguous_byte_ container and route them through an iterator_input_adapter built from data()/data()+size(). The generic iterator-based container overload is constrained to exclude these, so the two overloads are disjoint and there is no ambiguity (a plain competing overload loses to the greedy forwarding-reference container overload on reference binding, and a factory partial-specialization is ambiguous - both were tried and rejected). The pointer keeps the container's own element type, so char_type - and therefore all parsing behavior - is byte-for-byte identical to the iterator path (const char* for std::string, const std::uint8_t* for std::vector); only the raw pointer additionally turns on the fast paths. Lifetimes are unchanged: the container outlives the adapter for the full parse expression, exactly as the iterators it replaces did. Measured, C++11, json::parse/accept(std::string), g++ 13: long ASCII strings: accept 201 -> 3200 MB/s (~16x), parse 174 -> 1444 dense CJK: accept 263 -> 697 MB/s (~2.6x) short strings: accept 163 -> 243 MB/s (~1.5x) Verified: char_type preserved for std::string (char) and std::vector (uint8_t); CBOR/MsgPack round-trips from std::vector unchanged; 1,000,000 randomized documents accept and parse identically via std::string and via std::istream; deserialization/user-defined-input/parser/lexer/conversions/diagnostic- position suites pass (20,480 assertions); warning-clean on g++ and clang in C++11/17/20. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01AXcDtEma2PjxgmPS9cQGzA Signed-off-by: Niels Lohmann * Add optional simdutf backend for bulk UTF-8 validation (JSON_USE_SIMDUTF) The bulk string scanner validates UTF-8 straight from a contiguous buffer. The scalar validator caps at ~0.3-0.7 GB/s on non-ASCII text; a SIMD validator reaches several GB/s. Rather than hand-rolling SIMD UTF-8 validation (easy to get subtly wrong - a from-scratch SSE attempt rejected valid CJK), wire in the vetted simdutf library behind an opt-in switch. simdutf is not header-only (it ships simdutf.cpp and uses runtime CPU dispatch), so it is not vendored: defining JSON_USE_SIMDUTF includes and routes the bulk validator through simdutf::validate_utf8; the project supplies and links simdutf. Undefined (the default), nothing external is included and the portable C++11 scalar path is used, so the library stays header-only and its baseline behavior is unchanged. Design keeps behavior identical either way: - scan_string_bulk() now finds the run up to the next quote/escape/control byte (non-ASCII allowed) and validates it in one shot; on the rare validation failure it recomputes the exact valid prefix with the scalar helper, so ill-formed input still falls through to the byte path and is reported at the same position with the same message. - the per-sequence scalar path is factored into scalar_string_bulk_run() and is the default backend; the refactor is behavior-preserving and does not change scalar throughput. Verified: default and JSON_USE_SIMDUTF builds accept/reject/parse identically across 2,000,000 arbitrary-byte documents and 1,000,000 mixed-escape/UTF-8 documents (differential fuzz vs the streaming byte path); lexer/parser/diagnostic-position/deserialization suites pass under both configurations (20,188 assertions with the backend enabled); warning-clean on g++ and clang, C++11 and C++20, both configurations. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01AXcDtEma2PjxgmPS9cQGzA Signed-off-by: Niels Lohmann * Add a contiguous fast path for scanning numbers scan_number() reads a number one character at a time through the input adapter (get()) and appends each byte to token_buffer (add()) before converting. For contiguous input, the per-character get()/add() overhead dominates: it is roughly two thirds of the time spent on number-heavy parsing, far more than the value conversion itself. Add scan_number_bulk_contiguous(), which parses the whole number token straight from the input buffer: it validates and classifies the extent with the same grammar as scan_number()'s state machine, materializes token_buffer in one copy (substituting the locale decimal point exactly as scan_number() does), advances the adapter, and reuses the shared convert_number() tail. On anything it does not recognize as a well-formed number it makes no state change and returns token_type::uninitialized, so the caller falls back to scan_number(), which then produces the exact diagnostic. Errors and their positions are therefore unchanged. The conversion tail is factored out of scan_number() into convert_number() so both scanners share it; the fast path is selected by tag dispatch on the existing bulk_scan capability, so streaming/wide/user adapters are unaffected. Measured on pointer input, g++ 13 -O3: - integers: parse +65%, accept +98% - floats: parse +39%, accept +70% Verified: 2,000,000 randomized number documents (including overflow-range integers, long digit strings and %.17g doubles) parse identically via the contiguous path and the streaming byte path, matching value, type and round-trip text; the locale suite and existing parser/lexer/conversions/ deserialization tests pass; a new "lexer number fast path" test checks contiguous-vs-streaming parity, token classification, and that malformed numbers are rejected identically. Pure C++11, no intrinsics. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01AXcDtEma2PjxgmPS9cQGzA Signed-off-by: Niels Lohmann * Move byte-level scan/parse helpers out of lexer.hpp lexer.hpp had grown by ~600 lines of byte-level helpers that have no dependency on the lexer's template parameters and clutter the state machine. Move them, unchanged, into two focused headers as free functions in namespace detail: - number_parse.hpp: parse_integer_unsigned/parse_integer_signed (now templated on the number type) and parse_float_fast (Clinger's exact double fast path, with the decimal point passed as an argument instead of read from a lexer member). - string_scan.hpp: the SWAR string helpers (is_string_special, swar_string_special, find_string_special, validate_one_utf8, scalar_string_bulk_run) and the backend-dispatched string_bulk_run, including the optional simdutf include and find_string_delimiter. lexer.hpp now includes these and calls the free functions; the methods that touch lexer state (scan_string, scan_number, scan_string_bulk, scan_number_bulk_contiguous, convert_number) stay put. This is a pure code move with no behavior change: lexer.hpp drops from 2357 to 1934 lines, the now-unused // includes are removed, and the free-function form makes the SWAR helpers reusable elsewhere (e.g. the serializer's string escaping). Verified: default and JSON_USE_SIMDUTF builds compile; 2,000,000 number and 2,000,000 arbitrary-byte-string differential-fuzz documents parse identically to before; lexer/parser/conversions/deserialization/locale/ diagnostic-position suites pass (20,576 assertions); warning-clean on g++ and clang in C++11/17/20; the amalgamation regenerates and passes check-amalgamation. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01AXcDtEma2PjxgmPS9cQGzA Signed-off-by: Niels Lohmann * Fix clang-tidy findings and document JSON_USE_SIMDUTF in the nav - number_parse.hpp: use std::array for the powers-of-ten table (avoid-c-arrays) and `auto` for the cast-initialized result (modernize-use-auto), matching the codebase style (cf. the serializer's utf8d table). Indexing casts keep the -Wsign-conversion build clean. - add JSON_USE_SIMDUTF to the mkdocs navigation so the macro page is reachable. No behavior change; clang-tidy is clean on the new headers and the amalgamation is regenerated. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01AXcDtEma2PjxgmPS9cQGzA Signed-off-by: Niels Lohmann * Fix number fast path for custom string types without assign() The contiguous number fast path materialized token_buffer with token_buffer.assign(data, len), but string_t is only required to provide the minimal interface the rest of the lexer uses (push_back, append, clear, operator[], ...). Custom string types such as the test's alt_string do not implement assign(), so scan_number_bulk_contiguous() failed to compile for them (unit-alt-string), breaking the gcc/clang standards and old-compiler CI jobs. reset() already clears token_buffer, so fill it with append() - which alt_string and std::string both provide and which the string fast path already relies on - instead of assign(). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01AXcDtEma2PjxgmPS9cQGzA Signed-off-by: Niels Lohmann * Satisfy clang-tidy: parenthesize math and drop unused forwarding reference The CI clang-tidy (newer than the locally available version) reported two additional checks on the new code: - readability-math-missing-parentheses: parenthesize the (a * b) + c digit accumulations in number_parse.hpp. - cppcoreguidelines-missing-std-forward: the contiguous-byte-container input_adapter overload took a forwarding reference but only reads data()/size() and never forwards it. It is already disjoint from the generic container overload via SFINAE, so a plain const& is correct and clearer (and keeps the container alive for the whole parse just as before). No behavior change; char_type and routing are unchanged (std::string and std::vector still take the pointer adapter with char/uint8_t char_type), CBOR/MsgPack round-trips and the 2M number fuzz still pass. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01AXcDtEma2PjxgmPS9cQGzA Signed-off-by: Niels Lohmann * Use std::from_chars (Eisel-Lemire) for float conversion when available The Clinger fast path is exact only for the "easy" subset (<=19 significant digits, |exp10| <= 22); high-precision and scientific floats fall through to strtod, where the failed Clinger attempt actually makes parsing a net loss. std::from_chars implements the Eisel-Lemire algorithm in modern standard libraries: locale-independent, correctly rounded, and fast over the whole value range. convert_number() now tries parse_float_from_chars() first (guarded by __cpp_lib_to_chars, so C++11 and libc++-without-float-support keep the Clinger + strtod path unchanged), then Clinger, then strtof. from_chars is used only when it consumes the entire token; a partial parse means a non-'.' locale decimal point, and an under-/overflow (result_out_of_range) also declines - in both cases the existing strtod fallback supplies the exact value and the well-defined +/-inf/0 the parser expects, side-stepping the P4168 divergence between implementations. float and long double now get the fast path too (Clinger was double-only). Measured, C++17, g++ 13 -O3, json::parse/accept: - canada-style floats: ~unchanged (Clinger already covered them) - high-precision (17 digits): parse 2.1x, accept 2.5x - scientific (17 digits + exp): parse 3.6x, accept 4.1x Verified: C++11 (Clinger/strtod) and C++17 (from_chars) parse every value - including subnormals, boundary values, and 1e9999/1e-9999 over-/underflow - to bit-identical results; 2M number-fuzz clean; conversions/deserialization/ locale/number-fast-path suites pass in both C++11 and C++17; clang-tidy clean; warning-clean on g++ and clang in C++11/17/20. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01AXcDtEma2PjxgmPS9cQGzA Signed-off-by: Niels Lohmann * Guard from_chars use on JSON_HAS_CPP_17, not just __cpp_lib_to_chars libstdc++ 15 defines __cpp_lib_to_chars even in C++14 mode (via bits/version.h pulled in by other headers), but is only included under JSON_HAS_CPP_17. That made parse_float_from_chars() reference std::from_chars without the header in C++14 builds, breaking gcc-latest, icpx, and the offline-testdata jobs. Gate the use on JSON_HAS_CPP_17 && __cpp_lib_to_chars so it matches the include condition exactly; C++11/14 always take the scalar fallback. Verified by forcing __cpp_lib_to_chars in a C++14 build: the guard suppresses std::from_chars and it compiles. C++17 behavior is unchanged. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01AXcDtEma2PjxgmPS9cQGzA Signed-off-by: Niels Lohmann * Disable Clinger float fast path under extended FP precision (x87) The contiguous number fast path uses a Clinger-style exact algorithm (significand * 10^scale in double arithmetic), which is only correctly rounded when double operations are evaluated in true 53-bit precision. On the x87 FPU used by 32-bit x86 (FLT_EVAL_METHOD == 2) the single multiply/divide is computed in 80-bit and then double-rounded to double, so a small fraction of values land 1 ULP off. This surfaced as test-cbor_cpp11 and test-msgpack_cpp11 failing on the mingw (x86) job for regression/floats.json: the C++17 builds pass because they take the correctly-rounded std::from_chars path, while C++11 falls back to parse_float_fast(). A 5M-sample check over shortest round-trip decimals reproduces it: 0 divergences with 53-bit doubles, ~1 in 25 000 with 80-bit intermediates; declining to std::strtod fixes all of them. Guard parse_float_fast() on FLT_EVAL_METHOD so it declines whenever the platform evaluates doubles in extended precision, letting the caller use the correctly-rounded std::from_chars / std::strtod path instead. On mainstream x86-64/ARM64 (FLT_EVAL_METHOD == 0) the fast path is unchanged. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01AXcDtEma2PjxgmPS9cQGzA Signed-off-by: Niels Lohmann * Guard include with __has_include for GCC 7 GCC 7 sets __cplusplus to the C++17 value under -std=gnu++1z, so JSON_HAS_CPP_17 is defined, but its libstdc++ ships no header (added in GCC 8; floating-point from_chars in GCC 11). The unconditional "#if defined(JSON_HAS_CPP_17) #include " therefore failed to compile there: "fatal error: charconv: No such file or directory" in the ci_test_compilers_gcc (7) job. Wrap the include in __has_include(), mirroring the library's existing handling of and in macro_scope.hpp. When the header is absent, __cpp_lib_to_chars stays undefined and parse_float_from_chars() takes its scalar fallback, so the from_chars use site (already gated on __cpp_lib_to_chars) is never reached. GCC 8-10, which have but no floating-point from_chars, are unaffected: they include the header but still take the fallback. GCC 11+ is unchanged. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01AXcDtEma2PjxgmPS9cQGzA Signed-off-by: Niels Lohmann * Restore the column when ungetting a newline The contiguous number fast path never reads the character that terminates a number token, while scan_number() reads it and then ungets it. When that character is a newline, get() has already cleared chars_read_current_line, and unget() could only restore lines_read - leaving the column at 0. The two paths therefore reported different columns for the same document: json::parse("[01\n]") -> line 1, column 3 json::parse(stringstream) -> line 1, column 0 Remember the column the newline was read at so unget() can restore it. Both paths now report the position the offending token actually starts at, which also fixes the pre-existing column-0 artifact for streaming input. Signed-off-by: Niels Lohmann * Extend the bulk scan fast paths to sized sentinels supports_bulk_scan required IteratorType and SentinelType to be the same type, which excluded std::counted_iterator paired with std::default_sentinel_t - the combination #5268 had already enabled for the memcpy fast path. Such input fell back to the byte-at-a-time scanner even though it is contiguous and its remaining length is computable in O(1). Factor the "distance is computable in O(1)" test into sentinel_is_sized and use it for iterator_is_contiguous, supports_seek, and supports_bulk_scan alike, and share the std::ranges::distance/std::distance dispatch through a remaining_count() helper. Signed-off-by: Niels Lohmann * Document JSON_USE_SIMDUTF on the macro overview page The macro was only listed in the API macro index; add it to the supported macros overview alongside the other JSON_USE_* macros, and note that it selects between two definitions of the same inline function and so must be defined identically in every translation unit. Signed-off-by: Niels Lohmann * Amalgamate source code Signed-off-by: Niels Lohmann * Cover the counted-iterator bulk scan paths Sized sentinels newly reach the bulk string/number scanners and the seek-based token reconstruction, so exercise both: - diagnostics that quote the offending token, which are rebuilt from the consumed input via copy_consumed_range() - inputs whose count ends before the underlying buffer does, including a closing quote that exists only behind the count, a cut inside an 8-byte SWAR stride, and a cut inside a UTF-8 sequence Signed-off-by: Niels Lohmann * Fix the SentinelType example on iterator_input_adapter The comment offered "a C++20 sentinel or counted_iterator" as examples of a SentinelType, but std::counted_iterator is the IteratorType - the sentinel it pairs with is std::default_sentinel_t. #5268 corrected the same wording in the API documentation and left the code comment behind. Signed-off-by: Niels Lohmann * Do not discard the parse result in the error position check json::parse is declared warn_unused_result, and CHECK_THROWS_WITH_AS evaluates its expression as a discarded statement, so the assertion broke the -Werror builds (GCC -Werror=unused-result, MSVC C4834 under /WX). Compare against the helper that already captures the message instead. Signed-off-by: Niels Lohmann * Lock the two number grammars together with a parity test The JSON number grammar is encoded twice: as the scan_number() state machine and as the contiguous fast path. The fast path declining on anything it does not recognize keeps most divergence harmless, but if it ever accepted something the state machine rejects the result would be a silent correctness bug, and the existing test only pinned a hand-written list of numbers. Enumerate every string of length 1..4 over "01.eE+-" (2800 tokens) and require both paths to agree on the parsed value and on the exact error message. Verified to fail if the fast path's grammar is perturbed. Signed-off-by: Niels Lohmann * Skip token_buffer for integers on the contiguous path An integer token does not need token_buffer: the number_integer and number_unsigned SAX callbacks take only the value, and the overflow diagnostic rebuilds the text from the input via get_token_string(). Convert straight from the input buffer and materialize the token only for the floating-point tail, which still needs a NUL-terminated buffer for strtod. JSON_DIAGNOSTIC_POSITIONS derives a number's start position from get_string().size(), so the copy is kept when that is enabled. The integer dispatch is factored into convert_integer() and shared with convert_number(), so both scanners keep using one implementation. Integer-heavy input, 400k values, -O3: parse accept gcc 16 +14% +23% clang +15% +22% Signed-off-by: Niels Lohmann * Amalgamate source code Signed-off-by: Niels Lohmann * Cover the bulk string and UTF-8 scanners These paths had no dedicated tests and rested on differential fuzzing only. Add three sections, all comparing the contiguous scanner against the byte-at-a-time one on the parsed value and on the exact error message: - every string of length 1..3 over an alphabet of ordinary ASCII, both specials, a control byte, escape characters, UTF-8 lead and continuation bytes, and a byte that is never valid - each at offset 0 and offset 9, so the bulk scanner sees them with and without a run behind them - every kind of run-ending byte at each offset across two 8-byte SWAR words, so multibyte sequences also straddle the word boundary - the boundaries of every range validate_one_utf8() recognizes: shortest and longest encodings, overlongs, both ends of the surrogate block, U+10FFFF and just past it, and truncated sequences Verified to fail if the bulk validator accepts surrogates, and if the SWAR word test stops detecting control characters. Signed-off-by: Niels Lohmann * Format the new string fast path test with astyle The pinned astyle expands a braced-init-list used as a range-for range onto several lines; hoist the two offsets into a named vector instead, which reads better and leaves nothing for astyle to reformat. Signed-off-by: Niels Lohmann * Fix shadowed locals and guard the exception-dependent tests Two problems in the tests added for the bulk scanners, both found by CI: - the inner `const json j` in the counted-iterator diagnostics shadowed the one declared at test-case scope, which -Wshadow rejects on GCC and clang and C4456 rejects on MSVC under /WX; rename them - the new parity checks parse deliberately invalid input, which calls std::abort() rather than throwing when JSON_NOEXCEPTION is defined, so they would have crashed the no-exception build; guard them the way the other tests do json::accept() does not abort, so the UTF-8 range assertions stay compiled without exceptions and keep covering validate_one_utf8() there. Signed-off-by: Niels Lohmann * Gate the C++20 iterator classification on JSON_HAS_RANGES Making supports_bulk_scan depend on iterator_is_contiguous meant the trait is now instantiated for every adapter, not only when get_elements() is called. On standard libraries with an incomplete that is fatal: libstdc++ 10 evaluates std::contiguous_iterator> by calling std::to_address, which needs an operator-> its counted_iterator does not have, so satisfaction checking is a hard error rather than false. Reported by clang 14 + libstdc++ 10. JSON_HAS_RANGES already encodes exactly this ("libstdc++ < 11 has incomplete C++20 ranges", #4440), so require it for the C++20 branch. Affected toolchains fall back to the pointer-only test and the byte-at-a-time scanner, which parses identically, just without the bulk fast paths. Signed-off-by: Niels Lohmann * Amalgamate source code Signed-off-by: Niels Lohmann * Satisfy clang-tidy in the new bulk scanner tests - give the helper lambdas an explicit std::string return type and return braced initializer lists (modernize-return-braced-init-list) - replace the C-style array of test cases with a std::vector (modernize-avoid-c-arrays) - silence pro-type-member-init on the two brace-initialized aggregates; default member initializers would stop them being aggregates in C++11 Signed-off-by: Niels Lohmann * Avoid escaped literals in the counted-iterator diagnostics list clang-tidy reads "[\"\\ud834\"]" as a literal better written raw, and the two literals written next to each other in "[\"a\x01""b\"]" as a missing comma. The concatenation was there to stop the hex escape swallowing the following character; build those documents from explicit bytes instead and use raw strings elsewhere. The byte sequences are unchanged. Signed-off-by: Niels Lohmann * Reattach convert_number's documentation @gregmarr spotted that convert_integer() was inserted between convert_number() and its doc block, leaving convert_integer() with two stacked blocks and convert_number() with none. Comment only; no code change. Signed-off-by: Niels Lohmann * Address review comments on the simdutf backend string_bulk_run() had the same `return scalar_string_bulk_run(...)` in both arms of the `#if`. The simdutf arm already falls through when validation fails, so a single return after the `#endif` says the same thing. The JSON_USE_SIMDUTF example showed `#include `, which string_scan.hpp already does under the same guard; users only have to put the header on the include path and link the library, not include it themselves. Set the version history entry to 3.13.0, matching the other macro pages documenting unreleased features. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01MXDi7NTMKAmoArUZSKMc4T Signed-off-by: Niels Lohmann * Pin the number error position against a non-newline terminator The comment claimed the reported column is the one the offending token starts at. It is the column reached after the token's last character - which is the actual point of the unget() change: a number terminated by a newline now reports what the same number terminated by a space always did. Assert that equality directly, and add a multi-character token where the start and end columns differ, so the invariant cannot be read off a single-character example. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01MXDi7NTMKAmoArUZSKMc4T Signed-off-by: Niels Lohmann * Do not repeat the integer conversion that just failed scan_number_bulk_contiguous() converts an integer token straight from the input buffer. When the value does not fit, it materializes token_buffer and calls convert_number(), which tried the very same integer conversion again before falling back to floating point. Recording the outcome in number_type skips the second attempt. The resulting token type and value are unchanged: convert_number() reached the float tail either way. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01MXDi7NTMKAmoArUZSKMc4T Signed-off-by: Niels Lohmann * Require a container's value_type to match what data() points at is_contiguous_byte_container accepted any type with a data() returning a pointer to a single-byte integral plus a size(). That is duck typing: the two members say nothing about size() counting the units data() points at. A type where it does not - fixed-size records, say - was routed to the pointer-based adapter and parsed as [data(), data() + size()) bytes, silently truncating input the iterator-based adapter had read in full: struct record_buffer { using value_type = std::array; std::string bytes; const char* data() const; // raw bytes std::size_t size() const; // in records const char* begin() const; const char* end() const; }; json::parse(record_buffer{"[1,2,3,4,5]"}); // parse error at column 3 Requiring the container's own value_type to be that same element type ties the two together. Every contiguous standard container satisfies it, so std::string, std::vector, std::array and std::string_view keep the fast path; anything else falls back to the iterator-based adapter, which is always correct. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01MXDi7NTMKAmoArUZSKMc4T Signed-off-by: Niels Lohmann * Only compile the simdutf backend from C++17 on simdutf.h rejects anything below C++17 with an #error, so defining JSON_USE_SIMDUTF in a C++11 or C++14 translation unit did not fail with a message about simdutf being unavailable - it failed to compile at all, taking the library's C++11 support with it. Nothing caught this because no build ever compiled that path. Gate the include and both uses on JSON_HAS_CPP_17, the same way number_parse.hpp gates std::from_chars. Below C++17 the macro now has no effect and the scalar validator runs; it accepts and rejects exactly the same input, so the macro is safe to set project-wide even when some translation units use an older standard. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01MXDi7NTMKAmoArUZSKMc4T Signed-off-by: Niels Lohmann * Test the simdutf backend in CI JSON_USE_SIMDUTF was documented and shipped but never compiled by anything in the repository, so nothing held the backend to the behavior the docs promise. Add JSON_TestSimdutf (OFF by default), which fetches simdutf and defines JSON_USE_SIMDUTF for every test target, and a ci_test_simdutf target that runs the whole suite in that configuration. Because simdutf needs C++17, the suite is built at C++11 as well, so one job covers both the scalar fallback with the macro defined and simdutf itself. The dependency hangs off test_main, whose usage requirements every test target inherits. The library target and the installed CMake package are deliberately untouched: making nlohmann_json link simdutf would put a find_dependency() in the exported package, which is a separate decision. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01MXDi7NTMKAmoArUZSKMc4T Signed-off-by: Niels Lohmann * Warn when JSON_TestSimdutf cannot reach the simdutf backend simdutf needs C++17: without it the dependency does not even compile, and with a C++17 compiler but no C++17-or-later standard under test it builds and then goes unused. Either way the option silently did nothing useful, or broke the configure step outright. Resolve the tested standards first, then check them: when none of them can reach simdutf, skip the dependency and say so, naming which of the two reasons applies and how to fix it. The tests then run against the scalar validator, which is what would have happened anyway. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01MXDi7NTMKAmoArUZSKMc4T Signed-off-by: Niels Lohmann * Use record_buffer::data() so clang does not flag it unneeded The record_buffer test type declares data() and size() so the is_contiguous_byte_container trait can see both and still reject the type on its value_type. data() was never called, so clang's -Wunneeded-member-function (under -Weverything -Werror) failed the C++20 build. Assert that data() points at the underlying bytes: it ODR-uses the member and documents the property the type is meant to demonstrate. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01XAYM1qhSA2FDaDcGfPW3fG Signed-off-by: Niels Lohmann * Skip the float fast path when it cannot succeed parse_float_fast() (Clinger) needs a significand below 2^53, so it always declines once the mantissa has 17 or more significant digits. convert_number() called it unconditionally, so those numbers were walked an extra time before strtod had to run anyway. On streaming input, where scanning is byte-at-a-time and there is no compensating win, that made canada.json about 6% slower than develop. Derive the significant-digit count from token_buffer indices - the digits are not scanned again - and skip the call when it is guaranteed to decline. Both scanners pass the offset where the mantissa ends; the count only has to be corrected for a leading "0", which the JSON grammar admits nowhere else. The integer path returns before the check, so integer-heavy input is unaffected. Values are unchanged: this only avoids an attempt that would have failed. Verified bit-exact against develop over every number in canada.json, floats.json, signed_ints.json, unsigned_ints.json, small_signed_ints.json, citm_catalog.json and twitter.json, for both the contiguous and the streaming scanner. parse, streaming develop before after canada.json 19.4ms 20.5ms 19.3ms floats.json 135.9ms 131.8ms 128.0ms parse, contiguous develop before after canada.json 15.5ms 12.9ms 11.7ms floats.json 98.6ms 69.8ms 66.7ms Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann Co-authored-by: Claude Opus 4.8 --- .github/workflows/ubuntu.yml | 2 +- cmake/ci.cmake | 18 + docs/mkdocs/docs/api/macros/index.md | 1 + .../docs/api/macros/json_use_simdutf.md | 71 ++ docs/mkdocs/docs/features/macros.md | 8 + docs/mkdocs/mkdocs.yml | 1 + .../nlohmann/detail/input/input_adapters.hpp | 152 ++- include/nlohmann/detail/input/lexer.hpp | 411 +++++- .../nlohmann/detail/input/number_parse.hpp | 302 +++++ include/nlohmann/detail/input/string_scan.hpp | 241 ++++ single_include/nlohmann/json.hpp | 1110 ++++++++++++++++- tests/CMakeLists.txt | 68 + tests/src/unit-class_lexer.cpp | 433 +++++++ tests/src/unit-user_defined_input.cpp | 239 ++++ 14 files changed, 2948 insertions(+), 109 deletions(-) create mode 100644 docs/mkdocs/docs/api/macros/json_use_simdutf.md create mode 100644 include/nlohmann/detail/input/number_parse.hpp create mode 100644 include/nlohmann/detail/input/string_scan.hpp diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 49ee9d3d6..69a3cbc45 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -100,7 +100,7 @@ jobs: container: ubuntu:focal strategy: matrix: - target: [ci_cmake_flags, ci_test_diagnostics, ci_test_diagnostic_positions, ci_test_noexceptions, ci_test_noimplicitconversions, ci_test_legacycomparison, ci_test_noglobaludls] + target: [ci_cmake_flags, ci_test_diagnostics, ci_test_diagnostic_positions, ci_test_noexceptions, ci_test_noimplicitconversions, ci_test_legacycomparison, ci_test_noglobaludls, ci_test_simdutf] steps: - name: Install build-essential run: apt-get update ; apt-get install -y build-essential unzip wget git libssl-dev diff --git a/cmake/ci.cmake b/cmake/ci.cmake index 6b1d325d8..18fef2075 100644 --- a/cmake/ci.cmake +++ b/cmake/ci.cmake @@ -212,6 +212,24 @@ add_custom_target(ci_test_legacycomparison COMMENT "Compile and test with legacy discarded value comparison enabled" ) +############################################################################### +# Validate UTF-8 with simdutf. +############################################################################### + +add_custom_target(ci_test_simdutf + COMMAND ${CMAKE_COMMAND} + -DCMAKE_BUILD_TYPE=Debug -GNinja + -DJSON_BuildTests=ON -DJSON_TestSimdutf=ON + # simdutf needs C++17, so the library falls back to its scalar validator + # below that: build the suite at C++11 to cover the fallback with the macro + # defined, and at C++17 to run every test against simdutf itself + "-DJSON_TestStandards=11\;17" + -S${PROJECT_SOURCE_DIR} -B${PROJECT_BINARY_DIR}/build_simdutf + COMMAND ${CMAKE_COMMAND} --build ${PROJECT_BINARY_DIR}/build_simdutf + COMMAND cd ${PROJECT_BINARY_DIR}/build_simdutf && ${CMAKE_CTEST_COMMAND} --parallel ${N} --output-on-failure + COMMENT "Compile and test with simdutf UTF-8 validation enabled" +) + ############################################################################### # Enable brace-init copy semantics. ############################################################################### diff --git a/docs/mkdocs/docs/api/macros/index.md b/docs/mkdocs/docs/api/macros/index.md index 507c04932..e818f032a 100644 --- a/docs/mkdocs/docs/api/macros/index.md +++ b/docs/mkdocs/docs/api/macros/index.md @@ -24,6 +24,7 @@ header. See also the [macro overview page](../../features/macros.md). - [**JSON_NO_IO**](json_no_io.md) - switch off functions relying on certain C++ I/O headers - [**JSON_SKIP_UNSUPPORTED_COMPILER_CHECK**](json_skip_unsupported_compiler_check.md) - do not warn about unsupported compilers - [**JSON_USE_GLOBAL_UDLS**](json_use_global_udls.md) - place user-defined string literals (UDLs) into the global namespace +- [**JSON_USE_SIMDUTF**](json_use_simdutf.md) - use the simdutf library to accelerate UTF-8 validation ## Library version diff --git a/docs/mkdocs/docs/api/macros/json_use_simdutf.md b/docs/mkdocs/docs/api/macros/json_use_simdutf.md new file mode 100644 index 000000000..611c1e9f1 --- /dev/null +++ b/docs/mkdocs/docs/api/macros/json_use_simdutf.md @@ -0,0 +1,71 @@ +# JSON_USE_SIMDUTF + +```cpp +#define JSON_USE_SIMDUTF +``` + +When defined, the parser validates the UTF-8 content of JSON strings that come from a **contiguous byte input** +(`std::string`, `std::vector`/``, string literals, `const char*` ranges, …) using the +[simdutf](https://github.com/simdutf/simdutf) library instead of the built-in scalar validator. On text with many +non-ASCII characters (e.g. CJK or emoji) this can validate several times faster. + +This is an **opt-in external dependency**. The library itself remains header-only and its behavior is unchanged: the +same input is accepted or rejected either way, and every parse error is reported at the same position with the same +message (simdutf is only used to fast-path *valid* runs; anything it flags falls back to the scalar path so the exact +diagnostic is preserved). Streaming inputs (files, `std::istream`, wide strings, user-defined adapters) always use the +scalar path. + +When `JSON_USE_SIMDUTF` is defined you must make the `simdutf.h` header available on the include path and link the +simdutf library. When it is not defined, no simdutf header is included and there is no dependency. + +!!! note "Requires C++17" + + simdutf requires C++17 and its header rejects older standards with an `#!cpp #error`. The backend is therefore only + compiled in from C++17 on. In C++11 and C++14 the macro has no effect and the scalar validator is used, which + accepts and rejects exactly the same input -- only throughput differs. Setting the macro project-wide is therefore + safe even when some translation units are built with an older standard. + +!!! warning "Define consistently" + + The macro selects between two definitions of the same inline validation function. It must therefore be defined + identically for **every** translation unit that includes the library; mixing translation units that define it with + ones that do not is an ODR violation. Prefer setting it as a compile definition on the target rather than with + `#!cpp #define` in individual source files. + +## Default definition + +By default, `#!cpp JSON_USE_SIMDUTF` is not defined and the portable C++11 scalar validator is used. + +```cpp +#undef JSON_USE_SIMDUTF +``` + +## Examples + +??? example + + The code below enables the simdutf backend for UTF-8 validation. + + ```cpp + #define JSON_USE_SIMDUTF 1 + #include + + ... + ``` + + The project must also link against simdutf, e.g. with CMake: + + ```cmake + target_compile_definitions(your_target PRIVATE JSON_USE_SIMDUTF) + target_link_libraries(your_target PRIVATE simdutf::simdutf) + ``` + +!!! hint "Testing this configuration" + + The unit tests can be built against the simdutf backend with the CMake option `JSON_TestSimdutf` (`OFF` by + default), which fetches simdutf and defines `JSON_USE_SIMDUTF` for every test target. The `ci_test_simdutf` target + runs the whole test suite in that configuration. + +## Version history + +- Added in version 3.13.0. diff --git a/docs/mkdocs/docs/features/macros.md b/docs/mkdocs/docs/features/macros.md index 1d169fdeb..c4602fa5a 100644 --- a/docs/mkdocs/docs/features/macros.md +++ b/docs/mkdocs/docs/features/macros.md @@ -137,6 +137,14 @@ behavior is deprecated and switched off (`0`) by default. See [full documentation of `JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON`](../api/macros/json_use_legacy_discarded_value_comparison.md). +## `JSON_USE_SIMDUTF` + +When defined, UTF-8 validation of JSON strings read from contiguous byte input is delegated to the +[simdutf](https://github.com/simdutf/simdutf) library instead of the built-in scalar validator. This is an opt-in +external dependency and is not defined by default. + +See [full documentation of `JSON_USE_SIMDUTF`](../api/macros/json_use_simdutf.md). + ## `NLOHMANN_DEFINE_TYPE_*(...)`, `NLOHMANN_DEFINE_DERIVED_TYPE_*(...)` The library defines 12 macros to simplify the serialization/deserialization of types. See the page on diff --git a/docs/mkdocs/mkdocs.yml b/docs/mkdocs/mkdocs.yml index 2e1337f47..ec3e462c1 100644 --- a/docs/mkdocs/mkdocs.yml +++ b/docs/mkdocs/mkdocs.yml @@ -296,6 +296,7 @@ nav: - 'JSON_USE_GLOBAL_UDLS': api/macros/json_use_global_udls.md - 'JSON_USE_IMPLICIT_CONVERSIONS': api/macros/json_use_implicit_conversions.md - 'JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON': api/macros/json_use_legacy_discarded_value_comparison.md + - 'JSON_USE_SIMDUTF': api/macros/json_use_simdutf.md - 'NLOHMANN_DEFINE_DERIVED_TYPE_INTRUSIVE, NLOHMANN_DEFINE_DERIVED_TYPE_INTRUSIVE_WITH_DEFAULT, NLOHMANN_DEFINE_DERIVED_TYPE_INTRUSIVE_ONLY_SERIALIZE, NLOHMANN_DEFINE_DERIVED_TYPE_NON_INTRUSIVE, NLOHMANN_DEFINE_DERIVED_TYPE_NON_INTRUSIVE_WITH_DEFAULT, NLOHMANN_DEFINE_DERIVED_TYPE_NON_INTRUSIVE_ONLY_SERIALIZE': api/macros/nlohmann_define_derived_type.md - 'NLOHMANN_DEFINE_TYPE_INTRUSIVE, NLOHMANN_DEFINE_TYPE_INTRUSIVE_WITH_DEFAULT, NLOHMANN_DEFINE_TYPE_INTRUSIVE_ONLY_SERIALIZE': api/macros/nlohmann_define_type_intrusive.md - 'NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE, NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT, NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_ONLY_SERIALIZE': api/macros/nlohmann_define_type_non_intrusive.md diff --git a/include/nlohmann/detail/input/input_adapters.hpp b/include/nlohmann/detail/input/input_adapters.hpp index ba8df07a6..bd19d32a8 100644 --- a/include/nlohmann/detail/input/input_adapters.hpp +++ b/include/nlohmann/detail/input/input_adapters.hpp @@ -155,11 +155,31 @@ class input_stream_adapter // General-purpose iterator-based adapter. It might not be as fast as // theoretically possible for some containers, but it is extremely versatile. -// SentinelType defaults to IteratorType for backward compatibility, but may -// be a different type (e.g., a C++20 sentinel or counted_iterator). +// SentinelType defaults to IteratorType for backward compatibility, but may be +// a different type, e.g. a C++20 sentinel such as std::default_sentinel_t when +// IteratorType is a std::counted_iterator. template class iterator_input_adapter { + // Whether the number of elements between two positions can be computed in + // O(1): either the iterator and the sentinel have the same type (plain + // std::distance) or, in C++20, the sentinel is a sized sentinel for the + // iterator (std::ranges::distance), e.g. std::default_sentinel_t paired + // with std::counted_iterator. + // + // JSON_HAS_RANGES gates the C++20 branch: on standard libraries with an + // incomplete (libstdc++ < 11, see #4440) evaluating + // std::contiguous_iterator on a std::counted_iterator is a hard error + // instead of yielding false, and these traits are instantiated for every + // adapter. Such toolchains fall back to the pointer-only test and simply + // use the byte-at-a-time scanner. + static constexpr bool sentinel_is_sized = +#if JSON_HAS_RANGES && defined(__cpp_lib_concepts) && defined(JSON_HAS_CPP_20) + std::is_same::value || std::sized_sentinel_for; +#else + std::is_same::value; +#endif + public: using char_type = typename std::iterator_traits::value_type; @@ -171,7 +191,7 @@ class iterator_input_adapter // in wide_string_input_adapter, which does not expose this). static constexpr bool supports_seek = std::is_same::iterator_category, std::random_access_iterator_tag>::value - && std::is_same::value + && sentinel_is_sized && sizeof(char_type) == 1; iterator_input_adapter(IteratorType first, SentinelType last) @@ -219,30 +239,60 @@ class iterator_input_adapter private: // whether IteratorType refers to a contiguous range and therefore supports // a std::memcpy fast path (pointers always do; in C++20 we can also detect - // library iterators such as those of std::vector and std::string). - // Computing the available element count needs either same-type iterators - // (plain std::distance) or, in C++20, a sized sentinel (std::ranges::distance), - // e.g. std::counted_iterator paired with std::default_sentinel_t. - static constexpr bool iterator_is_contiguous = -#if defined(__cpp_lib_concepts) && defined(JSON_HAS_CPP_20) - (std::is_same::value || std::sized_sentinel_for) - && (std::contiguous_iterator || std::is_pointer::value); + // library iterators such as those of std::vector and std::string). The + // available element count must also be computable in O(1), hence + // sentinel_is_sized. + static constexpr bool iterator_is_contiguous = sentinel_is_sized && +#if JSON_HAS_RANGES && defined(__cpp_lib_concepts) && defined(JSON_HAS_CPP_20) + (std::contiguous_iterator || std::is_pointer::value); #else - std::is_same::value && std::is_pointer::value; + std::is_pointer::value; #endif + // number of unread elements in [current, end) + std::size_t remaining_count() const + { +#if JSON_HAS_RANGES && defined(__cpp_lib_concepts) && defined(JSON_HAS_CPP_20) + // std::ranges::distance also supports sized sentinels of a different + // type (e.g. std::counted_iterator + std::default_sentinel_t) + return static_cast(std::ranges::distance(current, end)); +#else + return static_cast(std::distance(current, end)); +#endif + } + + public: + // Whether the remaining input is a single contiguous block of 1-byte + // elements that the lexer can inspect directly (used for the SWAR string + // fast path). + static constexpr bool supports_bulk_scan = + iterator_is_contiguous && sizeof(char_type) == 1; + + // Pointer to the next unread element; only valid when bulk_remaining() > 0. + const char_type* bulk_data() const + { + return &*current; + } + + // Number of unread elements available as one contiguous block. + std::size_t bulk_remaining() const + { + return remaining_count(); + } + + // Consume @a n elements previously inspected via bulk_data(). + void bulk_skip(std::size_t n) + { + std::advance(current, static_cast::difference_type>(n)); + } + + private: // contiguous fast path: bulk copy the remaining range with std::memcpy template std::size_t get_elements_impl(T* dest, std::size_t count, std::true_type /*contiguous*/) { const std::size_t wanted = count * sizeof(T); -#if defined(__cpp_lib_concepts) && defined(JSON_HAS_CPP_20) - // std::ranges::distance also supports sized sentinels of a different - // type (e.g. std::counted_iterator + std::default_sentinel_t) - const std::size_t available = static_cast(std::ranges::distance(current, end)) * sizeof(char_type); -#else - const std::size_t available = static_cast(std::distance(current, end)) * sizeof(char_type); -#endif + const std::size_t available = remaining_count() * sizeof(char_type); const std::size_t copied = (std::min)(wanted, available); if (JSON_HEDLEY_LIKELY(copied != 0)) { @@ -570,6 +620,46 @@ typename iterator_input_adapter_factory::adapter_typ return factory_type::create(first, last); } +// The element type a container's data() points at, cv-qualifiers removed. +// Ill-formed - and therefore SFINAE-friendly - for types without data(). +template +using container_data_t = typename std::remove_cv().data()) >::type >::type; + +// The container's own element type, cv-qualifiers removed. It is looked up on +// the bare type so it is also found when ContainerType is deduced as a +// reference by the forwarding-reference overload below. +template +using container_value_t = typename std::remove_cv < + typename std::remove_cv::type>::type::value_type >::type; + +// Detect a container that stores its elements contiguously as single bytes +// (std::string, std::vector, std::array, +// std::string_view, ...). Such inputs are wrapped in a pointer-based adapter so +// they benefit from the contiguous fast paths (bulk string scanning, memcpy for +// binary formats) in every C++ standard - not only in C++20, where the standard +// library iterators model std::contiguous_iterator and are detected directly. +// +// data() and size() on their own would be duck typing: they say nothing about +// size() counting the units data() points at, and reading [data(), data() + +// size()) as bytes would be wrong for a type where it does not. Requiring the +// container's own value_type to be that same single-byte element ties the two +// together; every contiguous standard container satisfies it. Anything else +// keeps the iterator-based adapter, which is always correct - only slower. +template +struct is_contiguous_byte_container : std::false_type {}; + +template +struct is_contiguous_byte_container < ContainerType, void_t < + container_data_t, + container_value_t, +decltype(std::declval().size()) >> + : std::integral_constant < bool, + std::is_pointer().data())>::value&& + std::is_integral>::value&& + sizeof(container_data_t) == 1 && + std::is_same, container_value_t>::value > {}; + // Convenience shorthand from container to iterator // Enables ADL on begin(container) and end(container) // Encloses the using declarations in namespace for not to leak them to outside scope @@ -597,12 +687,32 @@ struct container_input_adapter_factory< ContainerType, } // namespace container_input_adapter_factory_impl -template -typename container_input_adapter_factory_impl::container_input_adapter_factory::adapter_type input_adapter(ContainerType&& container) +// General container path (iterator-based). Contiguous single-byte containers +// are excluded here and routed through the pointer-based overload below. +template < typename ContainerType, + enable_if_t < !is_contiguous_byte_container::value, int > = 0 > +typename container_input_adapter_factory_impl::container_input_adapter_factory::adapter_type input_adapter(ContainerType && container) { return container_input_adapter_factory_impl::container_input_adapter_factory::create(std::forward(container)); } +// Contiguous single-byte containers (std::string, std::vector, ...) are +// wrapped in a pointer-based adapter so the contiguous fast paths apply in every +// standard. The pointer keeps the container's own element type (const char* for +// std::string, const std::uint8_t* for std::vector, ...), so the +// resulting char_type - and therefore the parsing behavior - is byte-for-byte +// identical to the iterator-based path; only the raw pointer additionally +// enables the bulk fast paths. The container outlives the adapter for the whole +// parse (temporaries live until the end of the full expression), exactly as the +// iterators it replaces did. +template < typename ContainerType, + enable_if_t < is_contiguous_byte_container::value, int > = 0 > +auto input_adapter(const ContainerType& container) +-> decltype(input_adapter(container.data(), container.data() + container.size())) +{ + return input_adapter(container.data(), container.data() + container.size()); +} + // specialization for std::string using string_input_adapter_type = decltype(input_adapter(std::declval())); diff --git a/include/nlohmann/detail/input/lexer.hpp b/include/nlohmann/detail/input/lexer.hpp index c241e793b..bc31337f9 100644 --- a/include/nlohmann/detail/input/lexer.hpp +++ b/include/nlohmann/detail/input/lexer.hpp @@ -19,7 +19,9 @@ #include // vector #include +#include #include +#include #include #include @@ -125,6 +127,25 @@ constexpr bool input_adapter_supports_seek(std::false_type /*detected*/) return false; } +// Detect whether an input adapter exposes a contiguous byte block that the +// lexer can scan directly (see iterator_input_adapter::supports_bulk_scan). +// Adapters without the flag - file, stream, wide-string, user-defined - fall +// back to the character-at-a-time string scanner. +template +using detect_supports_bulk_scan = decltype(InputAdapterType::supports_bulk_scan); + +template +constexpr bool input_adapter_supports_bulk_scan(std::true_type /*detected*/) +{ + return InputAdapterType::supports_bulk_scan; +} + +template +constexpr bool input_adapter_supports_bulk_scan(std::false_type /*detected*/) +{ + return false; +} + /*! @brief lexical analysis @@ -146,6 +167,14 @@ class lexer : public lexer_base static constexpr bool lazy_token_string = input_adapter_supports_seek(is_detected {}); + /// whether string scanning may bulk-consume runs of ordinary characters + /// directly from a contiguous input buffer (SWAR fast path). This requires + /// the token to be reconstructible lazily (lazy_token_string), so bypassing + /// the per-character capture in get() cannot lose error diagnostics. + static constexpr bool bulk_scan = + lazy_token_string + && input_adapter_supports_bulk_scan(is_detected {}); + public: using token_type = typename lexer_base::token_type; @@ -266,6 +295,40 @@ class lexer : public lexer_base return true; } + /// contiguous input: bulk-append the run of ordinary characters and complete + /// well-formed UTF-8 sequences starting at the current read position, leaving + /// the first byte that needs individual handling (the closing quote, an + /// escape, a control character, or an ill-formed UTF-8 byte) for get() + void scan_string_bulk(std::true_type /*bulk*/) + { + // a pending unget must be consumed through the normal path first + if (next_unget) + { + return; + } + const std::size_t remaining = ia.bulk_remaining(); + if (remaining == 0) + { + return; + } + const auto* const data = reinterpret_cast(ia.bulk_data()); + + const std::size_t pos = string_bulk_run(data, remaining); + if (pos == 0) + { + return; + } + token_buffer.append(reinterpret_cast(data), pos); + ia.bulk_skip(pos); + // the run contains no newline (all bytes < 0x20 are treated as special), + // so only the flat character counters advance + position.chars_read_total += pos; + position.chars_read_current_line += pos; + } + + /// streaming input: no bulk fast path + void scan_string_bulk(std::false_type /*bulk*/) const noexcept {} + /*! @brief scan a string literal @@ -291,6 +354,10 @@ class lexer : public lexer_base while (true) { + // bulk-consume ordinary characters from contiguous input, then + // handle the next special byte through the switch below + scan_string_bulk(std::integral_constant {}); + // get the next character switch (get()) { @@ -1009,6 +1076,12 @@ class lexer : public lexer_base // changed if minus sign, decimal point, or exponent is read token_type number_type = token_type::value_unsigned; + // offset just past the last mantissa byte in token_buffer (i.e. the + // index of 'e'/'E', or the whole token when there is no exponent). + // convert_number() uses it to count significant digits; npos means + // "not seen an exponent yet" and is resolved at scan_number_done + std::size_t mantissa_end = std::string::npos; + // state (init): we just found out we need to scan a number switch (current) { @@ -1194,6 +1267,9 @@ scan_number_decimal2: scan_number_exponent: // we just parsed an exponent number_type = token_type::value_float; + // this label is reached only right after the 'e'/'E' was appended (from + // the zero, any1, and decimal2 states), so the mantissa ends before it + mantissa_end = token_buffer.size() - 1; switch (get()) { case '+': @@ -1280,6 +1356,116 @@ scan_number_done: // we are done scanning a number) unget(); + // no exponent was scanned: the mantissa spans the whole token + if (mantissa_end == std::string::npos) + { + mantissa_end = token_buffer.size(); + } + + return convert_number(number_type, mantissa_end); + } + + /*! + @brief convert an already-validated integer token to its value + + The digit sequence in [first, last) has been validated by the caller, so a + dedicated parser can avoid the locale/errno overhead of std::strtoull. + + @return the token type on success; token_type::uninitialized if @a + number_type is not an integer type or the value does not fit, in + which case the caller falls back to the floating-point conversion + (matching the previous std::strtoull/std::strtoll behavior) + */ + token_type convert_integer(token_type number_type, const char* first, const char* last) + { + if (number_type == token_type::value_unsigned) + { + if (parse_integer_unsigned(first, last, value_unsigned)) + { + return token_type::value_unsigned; + } + } + else if (number_type == token_type::value_integer) + { + if (parse_integer_signed(first, last, value_integer)) + { + return token_type::value_integer; + } + } + + return token_type::uninitialized; + } + + /*! + @brief check whether Clinger's fast path can still succeed for this token + + parse_float_fast() needs a significand below 2^53. A mantissa with 17 or + more significant digits is at least 10^16 and therefore always exceeds it, + so calling the fast path would walk the token one extra time only to + decline before strtod has to run anyway. + + Significant digits are the mantissa's digits from the first nonzero one on; + the sign, the decimal point, leading zeros, and the exponent do not count. + The answer is derived from indices - the digits are not scanned again - so + this stays off the hot path of the number scanners. + + @param[in] mantissa_end offset just past the last mantissa byte in + token_buffer + @return false if parse_float_fast() is guaranteed to decline + */ + bool mantissa_fits_clinger(std::size_t mantissa_end) const + { + // 10^16 already exceeds 2^53, so 17 digits can never fit + constexpr std::size_t limit = 17; + + const std::size_t neg = (!token_buffer.empty() && token_buffer[0] == '-') ? 1u : 0u; + const std::size_t has_dot = (decimal_point_position != std::string::npos) ? 1u : 0u; + // the JSON grammar restricts the integer part to "0" or [1-9][0-9]*, so + // a leading zero can only be a lone "0", which is not significant + const std::size_t lead_zero = (token_buffer[neg] == '0') ? 1u : 0u; + JSON_ASSERT(mantissa_end >= neg + has_dot + lead_zero); + std::size_t digits = mantissa_end - neg - has_dot - lead_zero; + + if (JSON_HEDLEY_LIKELY(digits < limit)) + { + return true; + } + + // Only a number below 1 can carry further insignificant zeros, and only + // while the count stays at the limit does removing them change the + // answer - so this loop is skipped for all but a few tokens. Note + // token_buffer holds the locale's decimal point, so the fraction is + // located through decimal_point_position rather than by searching '.'. + if (lead_zero != 0) + { + JSON_ASSERT(has_dot != 0); // an integer "0" cannot reach the limit + for (std::size_t i = decimal_point_position + 1; + digits >= limit && i < mantissa_end && token_buffer[i] == '0'; ++i) + { + --digits; + } + } + + return digits < limit; + } + + /*! + @brief convert the number text in token_buffer to its value and token type + + The digit sequence in token_buffer has already been validated (by the + scan_number() state machine or by the contiguous fast path) and holds the + locale decimal point in place of '.'. Integers are parsed first and fall + back to floating point on overflow. This is shared so both scanners produce + identical results. + + @param[in] mantissa_end offset just past the last mantissa byte in + token_buffer (the index of 'e'/'E', or + token_buffer.size() when there is no exponent); + used to skip Clinger's fast path when it cannot + possibly succeed - see mantissa_fits_clinger() + */ + token_type convert_number(token_type number_type, std::size_t mantissa_end) + { // If the caller does not need the converted value (only whether the // input is syntactically valid; see json_sax_acceptor/accept()), an // unsigned/integer token can be reported without calling @@ -1332,45 +1518,37 @@ scan_number_done: } } - char* endptr = nullptr; // NOLINT(misc-const-correctness,cppcoreguidelines-pro-type-vararg,hicpp-vararg) - errno = 0; + const char* const num_begin = token_buffer.data(); + const char* const num_end = num_begin + token_buffer.size(); - // try to parse integers first and fall back to floats - if (number_type == token_type::value_unsigned) + if (number_type != token_type::value_float) { - const auto x = std::strtoull(token_buffer.data(), &endptr, 10); - - // we checked the number format before - JSON_ASSERT(endptr == token_buffer.data() + token_buffer.size()); - - if (errno != ERANGE) + const token_type integer_result = convert_integer(number_type, num_begin, num_end); + if (integer_result != token_type::uninitialized) { - value_unsigned = static_cast(x); - if (value_unsigned == x) - { - return token_type::value_unsigned; - } - } - } - else if (number_type == token_type::value_integer) - { - const auto x = std::strtoll(token_buffer.data(), &endptr, 10); - - // we checked the number format before - JSON_ASSERT(endptr == token_buffer.data() + token_buffer.size()); - - if (errno != ERANGE) - { - value_integer = static_cast(x); - if (value_integer == x) - { - return token_type::value_integer; - } + return integer_result; } } // this code is reached if we parse a floating-point number or if an - // integer conversion above failed + // integer conversion above overflowed. Prefer std::from_chars + // (Eisel-Lemire, locale-independent, correctly rounded) when available; + // otherwise the exact Clinger fast path (double only); otherwise the + // locale-aware strtof/strtod. + if (parse_float_from_chars(num_begin, num_end, value_float)) + { + return token_type::value_float; + } + // Skipping a fast path that cannot succeed is lossless and saves a full + // extra pass over the token's bytes, which otherwise shows up on + // high-precision inputs such as canada.json + if (mantissa_fits_clinger(mantissa_end) + && parse_float_fast(num_begin, num_end, decimal_point_char, value_float)) + { + return token_type::value_float; + } + + char* endptr = nullptr; // NOLINT(misc-const-correctness,cppcoreguidelines-pro-type-vararg,hicpp-vararg) strtof(value_float, token_buffer.data(), &endptr); // we checked the number format before @@ -1379,6 +1557,158 @@ scan_number_done: return token_type::value_float; } + /*! + @brief contiguous fast path for scanning a number + + Parses the whole number token straight from the input buffer, avoiding the + per-character get()/add() of scan_number(). On success it fills token_buffer + (with the locale decimal point substituted, as scan_number() does) and + returns the token type. On anything it does not fully recognize as a + well-formed number it makes no state change and returns + token_type::uninitialized, so the caller falls back to scan_number(), which + then produces the exact diagnostic. @a current is the first digit or the + leading minus (already read); the remaining bytes are taken from the adapter. + */ + token_type scan_number_bulk_contiguous() + { + // a pending unget offsets the buffer position from current; fall back + if (next_unget) + { + return token_type::uninitialized; + } + const std::size_t rem = ia.bulk_remaining(); + if (rem == 0) + { + // the first digit is the last input byte; let scan_number() finish + return token_type::uninitialized; + } + // the byte before the next unread one is current (contiguous input) + const char* const data = reinterpret_cast(ia.bulk_data()) - 1; + const std::size_t avail = rem + 1; + + // validate + classify the number extent (mirrors scan_number()'s grammar) + std::size_t i = 0; + std::size_t dot_index = std::string::npos; + token_type number_type = token_type::value_unsigned; + if (data[0] == '-') + { + number_type = token_type::value_integer; + i = 1; + if (i >= avail) + { + return token_type::uninitialized; + } + } + if (data[i] == '0') + { + ++i; + } + else if (data[i] >= '1' && data[i] <= '9') + { + ++i; + while (i < avail && data[i] >= '0' && data[i] <= '9') + { + ++i; + } + } + else + { + return token_type::uninitialized; + } + if (i < avail && data[i] == '.') + { + number_type = token_type::value_float; + dot_index = i; + ++i; + if (i >= avail || !(data[i] >= '0' && data[i] <= '9')) + { + return token_type::uninitialized; + } + while (i < avail && data[i] >= '0' && data[i] <= '9') + { + ++i; + } + } + // the mantissa ends here, whether or not an exponent part follows + const std::size_t mantissa_end = i; + if (i < avail && (data[i] == 'e' || data[i] == 'E')) + { + number_type = token_type::value_float; + ++i; + if (i < avail && (data[i] == '+' || data[i] == '-')) + { + ++i; + } + if (i >= avail || !(data[i] >= '0' && data[i] <= '9')) + { + return token_type::uninitialized; + } + while (i < avail && data[i] >= '0' && data[i] <= '9') + { + ++i; + } + } + const std::size_t len = i; + + // reset() records where this token starts (for diagnostics), so it has + // to run before the input position advances below + reset(); + + // An integer token needs no token_buffer: the SAX callbacks for + // number_integer/number_unsigned take only the value, and the overflow + // diagnostic rebuilds the text from the input. Convert straight from the + // input buffer and leave token_buffer empty. (JSON_DIAGNOSTIC_POSITIONS + // derives a number's start position from get_string().size(), so there + // the token still has to be materialized.) +#if !JSON_DIAGNOSTIC_POSITIONS + if (number_type != token_type::value_float) + { + const token_type integer_result = convert_integer(number_type, data, data + len); + if (JSON_HEDLEY_LIKELY(integer_result != token_type::uninitialized)) + { + ia.bulk_skip(len - 1); + position.chars_read_total += (len - 1); + position.chars_read_current_line += (len - 1); + return integer_result; + } + // The value does not fit an integer, so this token converts as a + // float. Recording that here keeps convert_number() below from + // repeating the integer attempt that just failed. + number_type = token_type::value_float; + } +#endif + + // materialize the token exactly as scan_number() would, substituting the + // locale decimal point so convert_number()'s strtof fallback stays valid. + // reset() already cleared token_buffer, so append() fills it (assign() is + // avoided because custom string_t types need not provide it) + token_buffer.append(reinterpret_cast(data), len); + if (dot_index != std::string::npos) + { + token_buffer[dot_index] = static_cast(decimal_point_char); + decimal_point_position = dot_index; + } + + ia.bulk_skip(len - 1); + position.chars_read_total += (len - 1); + position.chars_read_current_line += (len - 1); + + return convert_number(number_type, mantissa_end); + } + + /// contiguous input: try the number fast path, else the byte-path scanner + token_type scan_number_dispatch(std::true_type /*bulk*/) + { + const token_type t = scan_number_bulk_contiguous(); + return (t != token_type::uninitialized) ? t : scan_number(); + } + + /// streaming input: always use the byte-path scanner + token_type scan_number_dispatch(std::false_type /*bulk*/) + { + return scan_number(); + } + /*! @param[in] literal_text the literal text to expect @param[in] length the length of the passed literal text @@ -1482,6 +1812,9 @@ scan_number_done: if (current == '\n') { ++position.lines_read; + // remember the column the newline was read at: chars_read_current_line + // is about to be cleared, and a matching unget() cannot reconstruct it + chars_read_before_newline = position.chars_read_current_line; position.chars_read_current_line = 0; } @@ -1538,12 +1871,20 @@ scan_number_done: --position.chars_read_total; // in case we "unget" a newline, we have to also decrement the lines_read + // and restore the column that get() cleared when it saw the newline; + // chars_read_current_line == 0 can only mean the last get() read one if (position.chars_read_current_line == 0) { if (position.lines_read > 0) { --position.lines_read; } + + // chars_read_before_newline counts the newline itself, which is the + // character being ungotten, hence the -1 + position.chars_read_current_line = (chars_read_before_newline > 0) + ? chars_read_before_newline - 1 + : 0; } else { @@ -1810,7 +2151,7 @@ scan_number_done: case '7': case '8': case '9': - return scan_number(); + return scan_number_dispatch(std::integral_constant {}); // end of input (the null byte is needed when parsing from // string literals) @@ -1841,6 +2182,10 @@ scan_number_done: /// the start position of the current token position_t position {}; + /// the value chars_read_current_line had when the last newline was read, so + /// that unget() can restore the column instead of leaving it at 0 + std::size_t chars_read_before_newline = 0; + /// raw input token string for error messages; only populated for streaming /// adapters (seekable adapters reconstruct it lazily via token_string_start) std::vector token_string {}; diff --git a/include/nlohmann/detail/input/number_parse.hpp b/include/nlohmann/detail/input/number_parse.hpp new file mode 100644 index 000000000..e50c3f67f --- /dev/null +++ b/include/nlohmann/detail/input/number_parse.hpp @@ -0,0 +1,302 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#pragma once + +#include // array +#include // FLT_EVAL_METHOD +#include // size_t +#include // int64_t, uint64_t +#include // numeric_limits + +#include + +// std::from_chars lives in , but being in C++17 mode does not +// guarantee the header exists: GCC 7 sets __cplusplus to C++17 yet ships no +// (added in GCC 8; floating-point support in GCC 11). Guard the +// include with __has_include so such toolchains fall back to the scalar path. +#if defined(JSON_HAS_CPP_17) && defined(__has_include) + #if __has_include() + #include // from_chars (only used when __cpp_lib_to_chars is defined) + #include // errc + #endif +#endif + +// This file contains the value-conversion helpers used by the lexer to turn an +// already-validated number token into a value, without the locale/errno +// overhead of std::strtoull/std::strtod. They are free functions so the lexer +// stays focused on scanning; see lexer::convert_number(). + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +/*! +@brief fast integer parser for an already-validated unsigned integer + +The number scanner has already checked that [first, last) is a valid JSON +integer, so this only needs to accumulate the digits and detect overflow. This +avoids the locale/errno machinery of std::strtoull, which dominates +integer-heavy inputs. + +@param[in] first pointer to the first character (a digit) +@param[in] last pointer past the last character +@param[out] value the parsed value on success +@return true if the value fit into @a NumberUnsignedType; false on overflow, in + which case the caller falls back to floating-point parsing (matching the + previous std::strtoull behavior) +*/ +template +bool parse_integer_unsigned(const char* first, const char* last, NumberUnsignedType& value) noexcept +{ + // accumulate in the widest unsigned type used by the previous strtoull + // path so the overflow behavior is unchanged for custom number types + std::uint64_t x = 0; + constexpr std::uint64_t cutoff = (std::numeric_limits::max)() / 10u; + constexpr std::uint64_t cutlim = (std::numeric_limits::max)() % 10u; + for (const char* p = first; p != last; ++p) + { + const auto digit = static_cast(static_cast(*p) - static_cast('0')); + if (JSON_HEDLEY_UNLIKELY(x > cutoff || (x == cutoff && digit > cutlim))) + { + return false; + } + x = (x * 10u) + digit; + } + value = static_cast(x); + // reject values that do not round-trip into a narrower NumberUnsignedType + return static_cast(value) == x; +} + +/*! +@brief fast integer parser for an already-validated negative integer + +@param[in] first pointer to the leading '-' +@param[in] last pointer past the last character +@param[out] value the parsed (negative) value on success +@return true on success; false on overflow (caller falls back to float) +*/ +template +bool parse_integer_signed(const char* first, const char* last, NumberIntegerType& value) noexcept +{ + // the state machine only reaches the signed path via a leading '-' + JSON_ASSERT(first != last && *first == '-'); + std::uint64_t magnitude = 0; + // |INT64_MIN| == INT64_MAX + 1; this is the largest admissible magnitude + constexpr std::uint64_t limit = static_cast((std::numeric_limits::max)()) + 1u; + for (const char* p = first + 1; p != last; ++p) + { + const auto digit = static_cast(static_cast(*p) - static_cast('0')); + if (JSON_HEDLEY_UNLIKELY(magnitude > (limit - digit) / 10u)) + { + return false; + } + magnitude = (magnitude * 10u) + digit; + } + const std::int64_t x = (magnitude == limit) + ? (std::numeric_limits::min)() + : -static_cast(magnitude); + value = static_cast(x); + // reject values that do not round-trip into a narrower NumberIntegerType + return static_cast(value) == x; +} + +/*! +@brief exact fast path for parsing a `double` (Clinger's algorithm) + +For the common case - at most 19 significant digits, a decimal exponent in +[-22, 22], and a significand below 2^53 - the value equals significand * +10^exp computed in IEEE-754 double arithmetic, which is exact under +round-to-nearest because both operands are exactly representable. This is the +same fast path used by fast_float/simdjson; the general cases are left to +std::strtod. The parser only activates for number_float_t == double; float and +long double keep the std::strtof/std::strtold paths (see the templated overload +below). + +@param[in] first pointer to the first character of the number +@param[in] last pointer past the last character +@param[in] decimal_point the (locale-dependent) decimal point character +@param[out] out the parsed value on success +@return true if the value was parsed exactly; false to fall back to strtod +*/ +template +bool parse_float_fast(const char* first, const char* last, DecimalPointType decimal_point, double& out) noexcept +{ +#if defined(FLT_EVAL_METHOD) && FLT_EVAL_METHOD != 0 + // Clinger's fast path is only exact when double operations are evaluated in + // true double precision. On platforms that keep intermediates in extended + // precision (e.g. the x87 FPU on 32-bit x86, where FLT_EVAL_METHOD == 2) the + // single significand * 10^scale step is double-rounded and can be 1 ULP off, + // so decline and let the caller fall back to the correctly-rounded + // std::from_chars / std::strtod path. + static_cast(first); + static_cast(last); + static_cast(decimal_point); + static_cast(out); + return false; +#else + static const std::array powers_of_ten = + { + { + 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, + 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22 + } + }; + + const char* p = first; + bool negative = false; + if (p != last && (*p == '-' || *p == '+')) + { + negative = (*p == '-'); + ++p; + } + + std::uint64_t significand = 0; + int num_digits = 0; + int fractional_digits = 0; + bool seen_dot = false; + bool any_digit = false; + for (; p != last; ++p) + { + const char c = *p; + if (c >= '0' && c <= '9') + { + any_digit = true; + if (JSON_HEDLEY_UNLIKELY(num_digits >= 19)) + { + return false; // significand may not fit into uint64_t + } + significand = (significand * 10u) + static_cast(c - '0'); + ++num_digits; + fractional_digits += static_cast(seen_dot); + } + else if (static_cast(c) == decimal_point) + { + if (JSON_HEDLEY_UNLIKELY(seen_dot)) + { + return false; + } + seen_dot = true; + } + else if (c == 'e' || c == 'E') + { + ++p; + break; + } + else + { + return false; + } + } + if (JSON_HEDLEY_UNLIKELY(!any_digit)) + { + return false; + } + + int exponent = 0; + if (p != last) // an exponent part remains + { + bool exp_negative = false; + if (p != last && (*p == '-' || *p == '+')) + { + exp_negative = (*p == '-'); + ++p; + } + bool any_exp_digit = false; + for (; p != last; ++p) + { + if (JSON_HEDLEY_UNLIKELY(*p < '0' || *p > '9')) + { + return false; + } + exponent = (exponent * 10) + (*p - '0'); + any_exp_digit = true; + if (JSON_HEDLEY_UNLIKELY(exponent > 9999)) + { + return false; + } + } + if (JSON_HEDLEY_UNLIKELY(!any_exp_digit)) + { + return false; + } + if (exp_negative) + { + exponent = -exponent; + } + } + + const int scale = exponent - fractional_digits; + if (JSON_HEDLEY_UNLIKELY(significand >= (static_cast(1) << 53))) + { + return false; // significand not exactly representable as double + } + + auto result = static_cast(significand); + if (scale >= 0) + { + if (JSON_HEDLEY_UNLIKELY(scale > 22)) + { + return false; + } + result *= powers_of_ten[static_cast(scale)]; + } + else + { + if (JSON_HEDLEY_UNLIKELY(-scale > 22)) + { + return false; + } + result /= powers_of_ten[static_cast(-scale)]; + } + out = negative ? -result : result; + return true; +#endif +} + +/// fast float path is only exact for `double`; decline for float/long double +template +bool parse_float_fast(const char* /*first*/, const char* /*last*/, DecimalPointType /*decimal_point*/, FloatType& /*out*/) noexcept +{ + return false; +} + +/*! +@brief parse a float with std::from_chars (Eisel-Lemire) when available + +std::from_chars is locale-independent, correctly rounded, and - via the +Eisel-Lemire algorithm in modern standard libraries - much faster than strtod +over the whole value range (not just the Clinger subset). It is used only when +__cpp_lib_to_chars indicates full floating-point support and only when it +consumes the entire token ([first, last)); a partial parse means the buffer +uses a non-'.' locale decimal point, in which case the caller falls back to the +locale-aware path. An under-/overflow (result_out_of_range) also declines, so +the caller's strtod fallback supplies the well-defined ±inf/0 result the parser +expects (side-stepping the P4168 divergence between implementations). + +@return true if the value was parsed exactly and fully; false to fall back +*/ +template +bool parse_float_from_chars(const char* first, const char* last, FloatType& out) noexcept +{ + // JSON_HAS_CPP_17 must gate the use as well as the include above: + // some standard libraries (e.g. libstdc++ 15) define __cpp_lib_to_chars even + // in C++14 mode, where is not included. +#if defined(JSON_HAS_CPP_17) && defined(__cpp_lib_to_chars) + const auto result = std::from_chars(first, last, out); + return result.ec == std::errc() && result.ptr == last; +#else + static_cast(first); + static_cast(last); + static_cast(out); + return false; +#endif +} + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END diff --git a/include/nlohmann/detail/input/string_scan.hpp b/include/nlohmann/detail/input/string_scan.hpp new file mode 100644 index 000000000..dc5b07a54 --- /dev/null +++ b/include/nlohmann/detail/input/string_scan.hpp @@ -0,0 +1,241 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#pragma once + +#include // size_t +#include // uint64_t +#include // memcpy + +#include + +// Optional SIMD backend for bulk UTF-8 validation. This is an opt-in external +// dependency: nlohmann/json itself stays header-only and the C++11 scalar +// validator below is always available; defining JSON_USE_SIMDUTF additionally +// requires the simdutf headers on the include path and linking the simdutf +// library. See string_bulk_run(). +// +// simdutf.h itself requires C++17 - it rejects older standards with an #error - +// so the backend is only compiled in from C++17 on. Below that the macro has no +// effect and the scalar validator is used; it accepts and rejects exactly the +// same input, so only throughput differs. macro_scope.hpp is included above to +// have JSON_HAS_CPP_17 available for this test. +#if defined(JSON_USE_SIMDUTF) && defined(JSON_HAS_CPP_17) + #include +#endif + +// This file contains the byte-level string-scanning helpers used by the lexer's +// contiguous fast path. They operate purely on raw bytes (no dependency on the +// lexer's template parameters) so they are free functions, keeping the lexer +// itself focused on the state machine; see lexer::scan_string_bulk(). + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +// classify a single byte as needing individual string handling: the closing +// quote, an escape, a control character, or a non-ASCII (UTF-8) +// lead/continuation byte. Ordinary bytes (0x20..0x7F except '"' and '\\') are +// copied verbatim, which the bulk scanner does 8 bytes at a time. +inline bool is_string_special(unsigned char c) noexcept +{ + return c == '\"' || c == '\\' || c < 0x20u || c >= 0x80u; +} + +// SWAR helper: return a word whose high bit is set in every byte of @a v that +// is_string_special(); zero if the 8 bytes are all ordinary. +inline std::uint64_t swar_string_special(std::uint64_t v) noexcept +{ + constexpr std::uint64_t ones = 0x0101010101010101ull; + constexpr std::uint64_t high = 0x8080808080808080ull; + const std::uint64_t q = v ^ 0x2222222222222222ull; // '"' (0x22) + const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; // '\\' (0x5C) + const std::uint64_t has_quote = (q - ones) & ~q & high; + const std::uint64_t has_backslash = (b - ones) & ~b & high; + const std::uint64_t has_control = (v - 0x2020202020202020ull) & ~v & high; // < 0x20 + const std::uint64_t has_non_ascii = v & high; // >= 0x80 + return has_quote | has_backslash | has_control | has_non_ascii; +} + +// return the index of the first is_string_special() byte in [data, data+n), or +// n if every byte is ordinary; scans 8 bytes at a time +inline std::size_t find_string_special(const unsigned char* data, std::size_t n) noexcept +{ + std::size_t i = 0; + for (; i + 8 <= n; i += 8) + { + std::uint64_t word = 0; + std::memcpy(&word, data + i, sizeof(word)); + if (swar_string_special(word) != 0) + { + // a special byte is in this word; locate it (endian-agnostic) + for (std::size_t j = 0; j < 8; ++j) + { + if (is_string_special(data[i + j])) + { + return i + j; + } + } + } + } + for (; i < n; ++i) + { + if (is_string_special(data[i])) + { + return i; + } + } + return n; +} + +// Validate one UTF-8 sequence at the front of [data, data+avail). Returns its +// length (2..4) only when the bytes form a *well-formed* sequence using exactly +// the same ranges as scan_string()'s per-byte switch, so the bulk path accepts +// precisely what the byte path accepts. Returns 0 for anything that is invalid, +// incomplete, or that the byte path must diagnose (the caller then defers to +// that path, keeping error messages unchanged). Lead bytes < 0x80 are handled +// by the caller and never passed here. +inline std::size_t validate_one_utf8(const unsigned char* data, std::size_t avail) noexcept +{ + const unsigned char c0 = data[0]; + if (c0 >= 0xC2 && c0 <= 0xDF) // U+0080..U+07FF + { + if (avail >= 2 && data[1] >= 0x80 && data[1] <= 0xBF) + { + return 2; + } + } + else if (c0 == 0xE0) // U+0800..U+0FFF + { + if (avail >= 3 && data[1] >= 0xA0 && data[1] <= 0xBF && data[2] >= 0x80 && data[2] <= 0xBF) + { + return 3; + } + } + else if ((c0 >= 0xE1 && c0 <= 0xEC) || c0 == 0xEE || c0 == 0xEF) // U+1000..U+CFFF, U+E000..U+FFFF + { + if (avail >= 3 && data[1] >= 0x80 && data[1] <= 0xBF && data[2] >= 0x80 && data[2] <= 0xBF) + { + return 3; + } + } + else if (c0 == 0xED) // U+D000..U+D7FF (excludes surrogates) + { + if (avail >= 3 && data[1] >= 0x80 && data[1] <= 0x9F && data[2] >= 0x80 && data[2] <= 0xBF) + { + return 3; + } + } + else if (c0 == 0xF0) // U+10000..U+3FFFF + { + if (avail >= 4 && data[1] >= 0x90 && data[1] <= 0xBF && data[2] >= 0x80 && data[2] <= 0xBF && data[3] >= 0x80 && data[3] <= 0xBF) + { + return 4; + } + } + else if (c0 >= 0xF1 && c0 <= 0xF3) // U+40000..U+FFFFF + { + if (avail >= 4 && data[1] >= 0x80 && data[1] <= 0xBF && data[2] >= 0x80 && data[2] <= 0xBF && data[3] >= 0x80 && data[3] <= 0xBF) + { + return 4; + } + } + else if (c0 == 0xF4) // U+100000..U+10FFFF + { + if (avail >= 4 && data[1] >= 0x80 && data[1] <= 0x8F && data[2] >= 0x80 && data[2] <= 0xBF && data[3] >= 0x80 && data[3] <= 0xBF) + { + return 4; + } + } + return 0; // invalid, incomplete, or must be diagnosed by the byte path +} + +// Scalar (C++11) computation of the bulk run length: the number of leading +// bytes in [data, data+n) that are ordinary ASCII or complete well-formed UTF-8 +// sequences, stopping before the first byte that needs individual handling (the +// closing quote, an escape, a control character, or an ill-formed/truncated +// sequence). ASCII is skipped 8 bytes at a time. +inline std::size_t scalar_string_bulk_run(const unsigned char* data, std::size_t n) noexcept +{ + std::size_t pos = 0; + while (pos < n) + { + pos += find_string_special(data + pos, n - pos); + if (pos >= n || data[pos] < 0x80u) + { + break; // end of buffer, or a quote/escape/control byte + } + const std::size_t seq = validate_one_utf8(data + pos, n - pos); + if (seq == 0) + { + break; // ill-formed or truncated: let the byte path diagnose it + } + pos += seq; + } + return pos; +} + +#if defined(JSON_USE_SIMDUTF) && defined(JSON_HAS_CPP_17) +// Index of the first quote/escape/control byte in [data, data+n) (non-ASCII +// bytes are *not* stops here - the whole run is handed to simdutf), or n. +inline std::size_t find_string_delimiter(const unsigned char* data, std::size_t n) noexcept +{ + constexpr std::uint64_t ones = 0x0101010101010101ull; + constexpr std::uint64_t high = 0x8080808080808080ull; + std::size_t i = 0; + for (; i + 8 <= n; i += 8) + { + std::uint64_t v = 0; + std::memcpy(&v, data + i, sizeof(v)); + const std::uint64_t q = v ^ 0x2222222222222222ull; + const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; + const std::uint64_t hit = ((q - ones) & ~q & high) + | ((b - ones) & ~b & high) + | ((v - 0x2020202020202020ull) & ~v & high); + if (hit != 0) + { + for (std::size_t j = 0; j < 8; ++j) + { + const unsigned char c = data[i + j]; + if (c == '\"' || c == '\\' || c < 0x20u) + { + return i + j; + } + } + } + } + for (; i < n; ++i) + { + const unsigned char c = data[i]; + if (c == '\"' || c == '\\' || c < 0x20u) + { + return i; + } + } + return n; +} +#endif + +// Backend-dispatched bulk run length. With JSON_USE_SIMDUTF the run up to the +// next delimiter is validated in one shot by simdutf; on the rare failure the +// scalar helper recomputes the exact valid prefix so the byte path still +// produces the precise diagnostic. Without it, the pure scalar path is used. +inline std::size_t string_bulk_run(const unsigned char* data, std::size_t n) noexcept +{ +#if defined(JSON_USE_SIMDUTF) && defined(JSON_HAS_CPP_17) + const std::size_t run = find_string_delimiter(data, n); + if (run != 0 && simdutf::validate_utf8(reinterpret_cast(data), run)) + { + return run; + } +#endif + return scalar_string_bulk_run(data, n); +} + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 1d1f290bc..10e7fff6f 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -7237,11 +7237,31 @@ class input_stream_adapter // General-purpose iterator-based adapter. It might not be as fast as // theoretically possible for some containers, but it is extremely versatile. -// SentinelType defaults to IteratorType for backward compatibility, but may -// be a different type (e.g., a C++20 sentinel or counted_iterator). +// SentinelType defaults to IteratorType for backward compatibility, but may be +// a different type, e.g. a C++20 sentinel such as std::default_sentinel_t when +// IteratorType is a std::counted_iterator. template class iterator_input_adapter { + // Whether the number of elements between two positions can be computed in + // O(1): either the iterator and the sentinel have the same type (plain + // std::distance) or, in C++20, the sentinel is a sized sentinel for the + // iterator (std::ranges::distance), e.g. std::default_sentinel_t paired + // with std::counted_iterator. + // + // JSON_HAS_RANGES gates the C++20 branch: on standard libraries with an + // incomplete (libstdc++ < 11, see #4440) evaluating + // std::contiguous_iterator on a std::counted_iterator is a hard error + // instead of yielding false, and these traits are instantiated for every + // adapter. Such toolchains fall back to the pointer-only test and simply + // use the byte-at-a-time scanner. + static constexpr bool sentinel_is_sized = +#if JSON_HAS_RANGES && defined(__cpp_lib_concepts) && defined(JSON_HAS_CPP_20) + std::is_same::value || std::sized_sentinel_for; +#else + std::is_same::value; +#endif + public: using char_type = typename std::iterator_traits::value_type; @@ -7253,7 +7273,7 @@ class iterator_input_adapter // in wide_string_input_adapter, which does not expose this). static constexpr bool supports_seek = std::is_same::iterator_category, std::random_access_iterator_tag>::value - && std::is_same::value + && sentinel_is_sized && sizeof(char_type) == 1; iterator_input_adapter(IteratorType first, SentinelType last) @@ -7301,30 +7321,60 @@ class iterator_input_adapter private: // whether IteratorType refers to a contiguous range and therefore supports // a std::memcpy fast path (pointers always do; in C++20 we can also detect - // library iterators such as those of std::vector and std::string). - // Computing the available element count needs either same-type iterators - // (plain std::distance) or, in C++20, a sized sentinel (std::ranges::distance), - // e.g. std::counted_iterator paired with std::default_sentinel_t. - static constexpr bool iterator_is_contiguous = -#if defined(__cpp_lib_concepts) && defined(JSON_HAS_CPP_20) - (std::is_same::value || std::sized_sentinel_for) - && (std::contiguous_iterator || std::is_pointer::value); + // library iterators such as those of std::vector and std::string). The + // available element count must also be computable in O(1), hence + // sentinel_is_sized. + static constexpr bool iterator_is_contiguous = sentinel_is_sized && +#if JSON_HAS_RANGES && defined(__cpp_lib_concepts) && defined(JSON_HAS_CPP_20) + (std::contiguous_iterator || std::is_pointer::value); #else - std::is_same::value && std::is_pointer::value; + std::is_pointer::value; #endif + // number of unread elements in [current, end) + std::size_t remaining_count() const + { +#if JSON_HAS_RANGES && defined(__cpp_lib_concepts) && defined(JSON_HAS_CPP_20) + // std::ranges::distance also supports sized sentinels of a different + // type (e.g. std::counted_iterator + std::default_sentinel_t) + return static_cast(std::ranges::distance(current, end)); +#else + return static_cast(std::distance(current, end)); +#endif + } + + public: + // Whether the remaining input is a single contiguous block of 1-byte + // elements that the lexer can inspect directly (used for the SWAR string + // fast path). + static constexpr bool supports_bulk_scan = + iterator_is_contiguous && sizeof(char_type) == 1; + + // Pointer to the next unread element; only valid when bulk_remaining() > 0. + const char_type* bulk_data() const + { + return &*current; + } + + // Number of unread elements available as one contiguous block. + std::size_t bulk_remaining() const + { + return remaining_count(); + } + + // Consume @a n elements previously inspected via bulk_data(). + void bulk_skip(std::size_t n) + { + std::advance(current, static_cast::difference_type>(n)); + } + + private: // contiguous fast path: bulk copy the remaining range with std::memcpy template std::size_t get_elements_impl(T* dest, std::size_t count, std::true_type /*contiguous*/) { const std::size_t wanted = count * sizeof(T); -#if defined(__cpp_lib_concepts) && defined(JSON_HAS_CPP_20) - // std::ranges::distance also supports sized sentinels of a different - // type (e.g. std::counted_iterator + std::default_sentinel_t) - const std::size_t available = static_cast(std::ranges::distance(current, end)) * sizeof(char_type); -#else - const std::size_t available = static_cast(std::distance(current, end)) * sizeof(char_type); -#endif + const std::size_t available = remaining_count() * sizeof(char_type); const std::size_t copied = (std::min)(wanted, available); if (JSON_HEDLEY_LIKELY(copied != 0)) { @@ -7652,6 +7702,46 @@ typename iterator_input_adapter_factory::adapter_typ return factory_type::create(first, last); } +// The element type a container's data() points at, cv-qualifiers removed. +// Ill-formed - and therefore SFINAE-friendly - for types without data(). +template +using container_data_t = typename std::remove_cv().data()) >::type >::type; + +// The container's own element type, cv-qualifiers removed. It is looked up on +// the bare type so it is also found when ContainerType is deduced as a +// reference by the forwarding-reference overload below. +template +using container_value_t = typename std::remove_cv < + typename std::remove_cv::type>::type::value_type >::type; + +// Detect a container that stores its elements contiguously as single bytes +// (std::string, std::vector, std::array, +// std::string_view, ...). Such inputs are wrapped in a pointer-based adapter so +// they benefit from the contiguous fast paths (bulk string scanning, memcpy for +// binary formats) in every C++ standard - not only in C++20, where the standard +// library iterators model std::contiguous_iterator and are detected directly. +// +// data() and size() on their own would be duck typing: they say nothing about +// size() counting the units data() points at, and reading [data(), data() + +// size()) as bytes would be wrong for a type where it does not. Requiring the +// container's own value_type to be that same single-byte element ties the two +// together; every contiguous standard container satisfies it. Anything else +// keeps the iterator-based adapter, which is always correct - only slower. +template +struct is_contiguous_byte_container : std::false_type {}; + +template +struct is_contiguous_byte_container < ContainerType, void_t < + container_data_t, + container_value_t, +decltype(std::declval().size()) >> + : std::integral_constant < bool, + std::is_pointer().data())>::value&& + std::is_integral>::value&& + sizeof(container_data_t) == 1 && + std::is_same, container_value_t>::value > {}; + // Convenience shorthand from container to iterator // Enables ADL on begin(container) and end(container) // Encloses the using declarations in namespace for not to leak them to outside scope @@ -7679,12 +7769,32 @@ struct container_input_adapter_factory< ContainerType, } // namespace container_input_adapter_factory_impl -template -typename container_input_adapter_factory_impl::container_input_adapter_factory::adapter_type input_adapter(ContainerType&& container) +// General container path (iterator-based). Contiguous single-byte containers +// are excluded here and routed through the pointer-based overload below. +template < typename ContainerType, + enable_if_t < !is_contiguous_byte_container::value, int > = 0 > +typename container_input_adapter_factory_impl::container_input_adapter_factory::adapter_type input_adapter(ContainerType && container) { return container_input_adapter_factory_impl::container_input_adapter_factory::create(std::forward(container)); } +// Contiguous single-byte containers (std::string, std::vector, ...) are +// wrapped in a pointer-based adapter so the contiguous fast paths apply in every +// standard. The pointer keeps the container's own element type (const char* for +// std::string, const std::uint8_t* for std::vector, ...), so the +// resulting char_type - and therefore the parsing behavior - is byte-for-byte +// identical to the iterator-based path; only the raw pointer additionally +// enables the bulk fast paths. The container outlives the adapter for the whole +// parse (temporaries live until the end of the full expression), exactly as the +// iterators it replaces did. +template < typename ContainerType, + enable_if_t < is_contiguous_byte_container::value, int > = 0 > +auto input_adapter(const ContainerType& container) +-> decltype(input_adapter(container.data(), container.data() + container.size())) +{ + return input_adapter(container.data(), container.data() + container.size()); +} + // specialization for std::string using string_input_adapter_type = decltype(input_adapter(std::declval())); @@ -7813,8 +7923,557 @@ NLOHMANN_JSON_NAMESPACE_END // #include +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // array +#include // FLT_EVAL_METHOD +#include // size_t +#include // int64_t, uint64_t +#include // numeric_limits + +// #include + + +// std::from_chars lives in , but being in C++17 mode does not +// guarantee the header exists: GCC 7 sets __cplusplus to C++17 yet ships no +// (added in GCC 8; floating-point support in GCC 11). Guard the +// include with __has_include so such toolchains fall back to the scalar path. +#if defined(JSON_HAS_CPP_17) && defined(__has_include) + #if __has_include() + #include // from_chars (only used when __cpp_lib_to_chars is defined) + #include // errc + #endif +#endif + +// This file contains the value-conversion helpers used by the lexer to turn an +// already-validated number token into a value, without the locale/errno +// overhead of std::strtoull/std::strtod. They are free functions so the lexer +// stays focused on scanning; see lexer::convert_number(). + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +/*! +@brief fast integer parser for an already-validated unsigned integer + +The number scanner has already checked that [first, last) is a valid JSON +integer, so this only needs to accumulate the digits and detect overflow. This +avoids the locale/errno machinery of std::strtoull, which dominates +integer-heavy inputs. + +@param[in] first pointer to the first character (a digit) +@param[in] last pointer past the last character +@param[out] value the parsed value on success +@return true if the value fit into @a NumberUnsignedType; false on overflow, in + which case the caller falls back to floating-point parsing (matching the + previous std::strtoull behavior) +*/ +template +bool parse_integer_unsigned(const char* first, const char* last, NumberUnsignedType& value) noexcept +{ + // accumulate in the widest unsigned type used by the previous strtoull + // path so the overflow behavior is unchanged for custom number types + std::uint64_t x = 0; + constexpr std::uint64_t cutoff = (std::numeric_limits::max)() / 10u; + constexpr std::uint64_t cutlim = (std::numeric_limits::max)() % 10u; + for (const char* p = first; p != last; ++p) + { + const auto digit = static_cast(static_cast(*p) - static_cast('0')); + if (JSON_HEDLEY_UNLIKELY(x > cutoff || (x == cutoff && digit > cutlim))) + { + return false; + } + x = (x * 10u) + digit; + } + value = static_cast(x); + // reject values that do not round-trip into a narrower NumberUnsignedType + return static_cast(value) == x; +} + +/*! +@brief fast integer parser for an already-validated negative integer + +@param[in] first pointer to the leading '-' +@param[in] last pointer past the last character +@param[out] value the parsed (negative) value on success +@return true on success; false on overflow (caller falls back to float) +*/ +template +bool parse_integer_signed(const char* first, const char* last, NumberIntegerType& value) noexcept +{ + // the state machine only reaches the signed path via a leading '-' + JSON_ASSERT(first != last && *first == '-'); + std::uint64_t magnitude = 0; + // |INT64_MIN| == INT64_MAX + 1; this is the largest admissible magnitude + constexpr std::uint64_t limit = static_cast((std::numeric_limits::max)()) + 1u; + for (const char* p = first + 1; p != last; ++p) + { + const auto digit = static_cast(static_cast(*p) - static_cast('0')); + if (JSON_HEDLEY_UNLIKELY(magnitude > (limit - digit) / 10u)) + { + return false; + } + magnitude = (magnitude * 10u) + digit; + } + const std::int64_t x = (magnitude == limit) + ? (std::numeric_limits::min)() + : -static_cast(magnitude); + value = static_cast(x); + // reject values that do not round-trip into a narrower NumberIntegerType + return static_cast(value) == x; +} + +/*! +@brief exact fast path for parsing a `double` (Clinger's algorithm) + +For the common case - at most 19 significant digits, a decimal exponent in +[-22, 22], and a significand below 2^53 - the value equals significand * +10^exp computed in IEEE-754 double arithmetic, which is exact under +round-to-nearest because both operands are exactly representable. This is the +same fast path used by fast_float/simdjson; the general cases are left to +std::strtod. The parser only activates for number_float_t == double; float and +long double keep the std::strtof/std::strtold paths (see the templated overload +below). + +@param[in] first pointer to the first character of the number +@param[in] last pointer past the last character +@param[in] decimal_point the (locale-dependent) decimal point character +@param[out] out the parsed value on success +@return true if the value was parsed exactly; false to fall back to strtod +*/ +template +bool parse_float_fast(const char* first, const char* last, DecimalPointType decimal_point, double& out) noexcept +{ +#if defined(FLT_EVAL_METHOD) && FLT_EVAL_METHOD != 0 + // Clinger's fast path is only exact when double operations are evaluated in + // true double precision. On platforms that keep intermediates in extended + // precision (e.g. the x87 FPU on 32-bit x86, where FLT_EVAL_METHOD == 2) the + // single significand * 10^scale step is double-rounded and can be 1 ULP off, + // so decline and let the caller fall back to the correctly-rounded + // std::from_chars / std::strtod path. + static_cast(first); + static_cast(last); + static_cast(decimal_point); + static_cast(out); + return false; +#else + static const std::array powers_of_ten = + { + { + 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, + 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22 + } + }; + + const char* p = first; + bool negative = false; + if (p != last && (*p == '-' || *p == '+')) + { + negative = (*p == '-'); + ++p; + } + + std::uint64_t significand = 0; + int num_digits = 0; + int fractional_digits = 0; + bool seen_dot = false; + bool any_digit = false; + for (; p != last; ++p) + { + const char c = *p; + if (c >= '0' && c <= '9') + { + any_digit = true; + if (JSON_HEDLEY_UNLIKELY(num_digits >= 19)) + { + return false; // significand may not fit into uint64_t + } + significand = (significand * 10u) + static_cast(c - '0'); + ++num_digits; + fractional_digits += static_cast(seen_dot); + } + else if (static_cast(c) == decimal_point) + { + if (JSON_HEDLEY_UNLIKELY(seen_dot)) + { + return false; + } + seen_dot = true; + } + else if (c == 'e' || c == 'E') + { + ++p; + break; + } + else + { + return false; + } + } + if (JSON_HEDLEY_UNLIKELY(!any_digit)) + { + return false; + } + + int exponent = 0; + if (p != last) // an exponent part remains + { + bool exp_negative = false; + if (p != last && (*p == '-' || *p == '+')) + { + exp_negative = (*p == '-'); + ++p; + } + bool any_exp_digit = false; + for (; p != last; ++p) + { + if (JSON_HEDLEY_UNLIKELY(*p < '0' || *p > '9')) + { + return false; + } + exponent = (exponent * 10) + (*p - '0'); + any_exp_digit = true; + if (JSON_HEDLEY_UNLIKELY(exponent > 9999)) + { + return false; + } + } + if (JSON_HEDLEY_UNLIKELY(!any_exp_digit)) + { + return false; + } + if (exp_negative) + { + exponent = -exponent; + } + } + + const int scale = exponent - fractional_digits; + if (JSON_HEDLEY_UNLIKELY(significand >= (static_cast(1) << 53))) + { + return false; // significand not exactly representable as double + } + + auto result = static_cast(significand); + if (scale >= 0) + { + if (JSON_HEDLEY_UNLIKELY(scale > 22)) + { + return false; + } + result *= powers_of_ten[static_cast(scale)]; + } + else + { + if (JSON_HEDLEY_UNLIKELY(-scale > 22)) + { + return false; + } + result /= powers_of_ten[static_cast(-scale)]; + } + out = negative ? -result : result; + return true; +#endif +} + +/// fast float path is only exact for `double`; decline for float/long double +template +bool parse_float_fast(const char* /*first*/, const char* /*last*/, DecimalPointType /*decimal_point*/, FloatType& /*out*/) noexcept +{ + return false; +} + +/*! +@brief parse a float with std::from_chars (Eisel-Lemire) when available + +std::from_chars is locale-independent, correctly rounded, and - via the +Eisel-Lemire algorithm in modern standard libraries - much faster than strtod +over the whole value range (not just the Clinger subset). It is used only when +__cpp_lib_to_chars indicates full floating-point support and only when it +consumes the entire token ([first, last)); a partial parse means the buffer +uses a non-'.' locale decimal point, in which case the caller falls back to the +locale-aware path. An under-/overflow (result_out_of_range) also declines, so +the caller's strtod fallback supplies the well-defined ±inf/0 result the parser +expects (side-stepping the P4168 divergence between implementations). + +@return true if the value was parsed exactly and fully; false to fall back +*/ +template +bool parse_float_from_chars(const char* first, const char* last, FloatType& out) noexcept +{ + // JSON_HAS_CPP_17 must gate the use as well as the include above: + // some standard libraries (e.g. libstdc++ 15) define __cpp_lib_to_chars even + // in C++14 mode, where is not included. +#if defined(JSON_HAS_CPP_17) && defined(__cpp_lib_to_chars) + const auto result = std::from_chars(first, last, out); + return result.ec == std::errc() && result.ptr == last; +#else + static_cast(first); + static_cast(last); + static_cast(out); + return false; +#endif +} + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + // #include +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // size_t +#include // uint64_t +#include // memcpy + +// #include + + +// Optional SIMD backend for bulk UTF-8 validation. This is an opt-in external +// dependency: nlohmann/json itself stays header-only and the C++11 scalar +// validator below is always available; defining JSON_USE_SIMDUTF additionally +// requires the simdutf headers on the include path and linking the simdutf +// library. See string_bulk_run(). +// +// simdutf.h itself requires C++17 - it rejects older standards with an #error - +// so the backend is only compiled in from C++17 on. Below that the macro has no +// effect and the scalar validator is used; it accepts and rejects exactly the +// same input, so only throughput differs. macro_scope.hpp is included above to +// have JSON_HAS_CPP_17 available for this test. +#if defined(JSON_USE_SIMDUTF) && defined(JSON_HAS_CPP_17) + #include +#endif + +// This file contains the byte-level string-scanning helpers used by the lexer's +// contiguous fast path. They operate purely on raw bytes (no dependency on the +// lexer's template parameters) so they are free functions, keeping the lexer +// itself focused on the state machine; see lexer::scan_string_bulk(). + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +// classify a single byte as needing individual string handling: the closing +// quote, an escape, a control character, or a non-ASCII (UTF-8) +// lead/continuation byte. Ordinary bytes (0x20..0x7F except '"' and '\\') are +// copied verbatim, which the bulk scanner does 8 bytes at a time. +inline bool is_string_special(unsigned char c) noexcept +{ + return c == '\"' || c == '\\' || c < 0x20u || c >= 0x80u; +} + +// SWAR helper: return a word whose high bit is set in every byte of @a v that +// is_string_special(); zero if the 8 bytes are all ordinary. +inline std::uint64_t swar_string_special(std::uint64_t v) noexcept +{ + constexpr std::uint64_t ones = 0x0101010101010101ull; + constexpr std::uint64_t high = 0x8080808080808080ull; + const std::uint64_t q = v ^ 0x2222222222222222ull; // '"' (0x22) + const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; // '\\' (0x5C) + const std::uint64_t has_quote = (q - ones) & ~q & high; + const std::uint64_t has_backslash = (b - ones) & ~b & high; + const std::uint64_t has_control = (v - 0x2020202020202020ull) & ~v & high; // < 0x20 + const std::uint64_t has_non_ascii = v & high; // >= 0x80 + return has_quote | has_backslash | has_control | has_non_ascii; +} + +// return the index of the first is_string_special() byte in [data, data+n), or +// n if every byte is ordinary; scans 8 bytes at a time +inline std::size_t find_string_special(const unsigned char* data, std::size_t n) noexcept +{ + std::size_t i = 0; + for (; i + 8 <= n; i += 8) + { + std::uint64_t word = 0; + std::memcpy(&word, data + i, sizeof(word)); + if (swar_string_special(word) != 0) + { + // a special byte is in this word; locate it (endian-agnostic) + for (std::size_t j = 0; j < 8; ++j) + { + if (is_string_special(data[i + j])) + { + return i + j; + } + } + } + } + for (; i < n; ++i) + { + if (is_string_special(data[i])) + { + return i; + } + } + return n; +} + +// Validate one UTF-8 sequence at the front of [data, data+avail). Returns its +// length (2..4) only when the bytes form a *well-formed* sequence using exactly +// the same ranges as scan_string()'s per-byte switch, so the bulk path accepts +// precisely what the byte path accepts. Returns 0 for anything that is invalid, +// incomplete, or that the byte path must diagnose (the caller then defers to +// that path, keeping error messages unchanged). Lead bytes < 0x80 are handled +// by the caller and never passed here. +inline std::size_t validate_one_utf8(const unsigned char* data, std::size_t avail) noexcept +{ + const unsigned char c0 = data[0]; + if (c0 >= 0xC2 && c0 <= 0xDF) // U+0080..U+07FF + { + if (avail >= 2 && data[1] >= 0x80 && data[1] <= 0xBF) + { + return 2; + } + } + else if (c0 == 0xE0) // U+0800..U+0FFF + { + if (avail >= 3 && data[1] >= 0xA0 && data[1] <= 0xBF && data[2] >= 0x80 && data[2] <= 0xBF) + { + return 3; + } + } + else if ((c0 >= 0xE1 && c0 <= 0xEC) || c0 == 0xEE || c0 == 0xEF) // U+1000..U+CFFF, U+E000..U+FFFF + { + if (avail >= 3 && data[1] >= 0x80 && data[1] <= 0xBF && data[2] >= 0x80 && data[2] <= 0xBF) + { + return 3; + } + } + else if (c0 == 0xED) // U+D000..U+D7FF (excludes surrogates) + { + if (avail >= 3 && data[1] >= 0x80 && data[1] <= 0x9F && data[2] >= 0x80 && data[2] <= 0xBF) + { + return 3; + } + } + else if (c0 == 0xF0) // U+10000..U+3FFFF + { + if (avail >= 4 && data[1] >= 0x90 && data[1] <= 0xBF && data[2] >= 0x80 && data[2] <= 0xBF && data[3] >= 0x80 && data[3] <= 0xBF) + { + return 4; + } + } + else if (c0 >= 0xF1 && c0 <= 0xF3) // U+40000..U+FFFFF + { + if (avail >= 4 && data[1] >= 0x80 && data[1] <= 0xBF && data[2] >= 0x80 && data[2] <= 0xBF && data[3] >= 0x80 && data[3] <= 0xBF) + { + return 4; + } + } + else if (c0 == 0xF4) // U+100000..U+10FFFF + { + if (avail >= 4 && data[1] >= 0x80 && data[1] <= 0x8F && data[2] >= 0x80 && data[2] <= 0xBF && data[3] >= 0x80 && data[3] <= 0xBF) + { + return 4; + } + } + return 0; // invalid, incomplete, or must be diagnosed by the byte path +} + +// Scalar (C++11) computation of the bulk run length: the number of leading +// bytes in [data, data+n) that are ordinary ASCII or complete well-formed UTF-8 +// sequences, stopping before the first byte that needs individual handling (the +// closing quote, an escape, a control character, or an ill-formed/truncated +// sequence). ASCII is skipped 8 bytes at a time. +inline std::size_t scalar_string_bulk_run(const unsigned char* data, std::size_t n) noexcept +{ + std::size_t pos = 0; + while (pos < n) + { + pos += find_string_special(data + pos, n - pos); + if (pos >= n || data[pos] < 0x80u) + { + break; // end of buffer, or a quote/escape/control byte + } + const std::size_t seq = validate_one_utf8(data + pos, n - pos); + if (seq == 0) + { + break; // ill-formed or truncated: let the byte path diagnose it + } + pos += seq; + } + return pos; +} + +#if defined(JSON_USE_SIMDUTF) && defined(JSON_HAS_CPP_17) +// Index of the first quote/escape/control byte in [data, data+n) (non-ASCII +// bytes are *not* stops here - the whole run is handed to simdutf), or n. +inline std::size_t find_string_delimiter(const unsigned char* data, std::size_t n) noexcept +{ + constexpr std::uint64_t ones = 0x0101010101010101ull; + constexpr std::uint64_t high = 0x8080808080808080ull; + std::size_t i = 0; + for (; i + 8 <= n; i += 8) + { + std::uint64_t v = 0; + std::memcpy(&v, data + i, sizeof(v)); + const std::uint64_t q = v ^ 0x2222222222222222ull; + const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; + const std::uint64_t hit = ((q - ones) & ~q & high) + | ((b - ones) & ~b & high) + | ((v - 0x2020202020202020ull) & ~v & high); + if (hit != 0) + { + for (std::size_t j = 0; j < 8; ++j) + { + const unsigned char c = data[i + j]; + if (c == '\"' || c == '\\' || c < 0x20u) + { + return i + j; + } + } + } + } + for (; i < n; ++i) + { + const unsigned char c = data[i]; + if (c == '\"' || c == '\\' || c < 0x20u) + { + return i; + } + } + return n; +} +#endif + +// Backend-dispatched bulk run length. With JSON_USE_SIMDUTF the run up to the +// next delimiter is validated in one shot by simdutf; on the rare failure the +// scalar helper recomputes the exact valid prefix so the byte path still +// produces the precise diagnostic. Without it, the pure scalar path is used. +inline std::size_t string_bulk_run(const unsigned char* data, std::size_t n) noexcept +{ +#if defined(JSON_USE_SIMDUTF) && defined(JSON_HAS_CPP_17) + const std::size_t run = find_string_delimiter(data, n); + if (run != 0 && simdutf::validate_utf8(reinterpret_cast(data), run)) + { + return run; + } +#endif + return scalar_string_bulk_run(data, n); +} + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + // #include // #include @@ -7922,6 +8581,25 @@ constexpr bool input_adapter_supports_seek(std::false_type /*detected*/) return false; } +// Detect whether an input adapter exposes a contiguous byte block that the +// lexer can scan directly (see iterator_input_adapter::supports_bulk_scan). +// Adapters without the flag - file, stream, wide-string, user-defined - fall +// back to the character-at-a-time string scanner. +template +using detect_supports_bulk_scan = decltype(InputAdapterType::supports_bulk_scan); + +template +constexpr bool input_adapter_supports_bulk_scan(std::true_type /*detected*/) +{ + return InputAdapterType::supports_bulk_scan; +} + +template +constexpr bool input_adapter_supports_bulk_scan(std::false_type /*detected*/) +{ + return false; +} + /*! @brief lexical analysis @@ -7943,6 +8621,14 @@ class lexer : public lexer_base static constexpr bool lazy_token_string = input_adapter_supports_seek(is_detected {}); + /// whether string scanning may bulk-consume runs of ordinary characters + /// directly from a contiguous input buffer (SWAR fast path). This requires + /// the token to be reconstructible lazily (lazy_token_string), so bypassing + /// the per-character capture in get() cannot lose error diagnostics. + static constexpr bool bulk_scan = + lazy_token_string + && input_adapter_supports_bulk_scan(is_detected {}); + public: using token_type = typename lexer_base::token_type; @@ -8063,6 +8749,40 @@ class lexer : public lexer_base return true; } + /// contiguous input: bulk-append the run of ordinary characters and complete + /// well-formed UTF-8 sequences starting at the current read position, leaving + /// the first byte that needs individual handling (the closing quote, an + /// escape, a control character, or an ill-formed UTF-8 byte) for get() + void scan_string_bulk(std::true_type /*bulk*/) + { + // a pending unget must be consumed through the normal path first + if (next_unget) + { + return; + } + const std::size_t remaining = ia.bulk_remaining(); + if (remaining == 0) + { + return; + } + const auto* const data = reinterpret_cast(ia.bulk_data()); + + const std::size_t pos = string_bulk_run(data, remaining); + if (pos == 0) + { + return; + } + token_buffer.append(reinterpret_cast(data), pos); + ia.bulk_skip(pos); + // the run contains no newline (all bytes < 0x20 are treated as special), + // so only the flat character counters advance + position.chars_read_total += pos; + position.chars_read_current_line += pos; + } + + /// streaming input: no bulk fast path + void scan_string_bulk(std::false_type /*bulk*/) const noexcept {} + /*! @brief scan a string literal @@ -8088,6 +8808,10 @@ class lexer : public lexer_base while (true) { + // bulk-consume ordinary characters from contiguous input, then + // handle the next special byte through the switch below + scan_string_bulk(std::integral_constant {}); + // get the next character switch (get()) { @@ -8806,6 +9530,12 @@ class lexer : public lexer_base // changed if minus sign, decimal point, or exponent is read token_type number_type = token_type::value_unsigned; + // offset just past the last mantissa byte in token_buffer (i.e. the + // index of 'e'/'E', or the whole token when there is no exponent). + // convert_number() uses it to count significant digits; npos means + // "not seen an exponent yet" and is resolved at scan_number_done + std::size_t mantissa_end = std::string::npos; + // state (init): we just found out we need to scan a number switch (current) { @@ -8991,6 +9721,9 @@ scan_number_decimal2: scan_number_exponent: // we just parsed an exponent number_type = token_type::value_float; + // this label is reached only right after the 'e'/'E' was appended (from + // the zero, any1, and decimal2 states), so the mantissa ends before it + mantissa_end = token_buffer.size() - 1; switch (get()) { case '+': @@ -9077,6 +9810,116 @@ scan_number_done: // we are done scanning a number) unget(); + // no exponent was scanned: the mantissa spans the whole token + if (mantissa_end == std::string::npos) + { + mantissa_end = token_buffer.size(); + } + + return convert_number(number_type, mantissa_end); + } + + /*! + @brief convert an already-validated integer token to its value + + The digit sequence in [first, last) has been validated by the caller, so a + dedicated parser can avoid the locale/errno overhead of std::strtoull. + + @return the token type on success; token_type::uninitialized if @a + number_type is not an integer type or the value does not fit, in + which case the caller falls back to the floating-point conversion + (matching the previous std::strtoull/std::strtoll behavior) + */ + token_type convert_integer(token_type number_type, const char* first, const char* last) + { + if (number_type == token_type::value_unsigned) + { + if (parse_integer_unsigned(first, last, value_unsigned)) + { + return token_type::value_unsigned; + } + } + else if (number_type == token_type::value_integer) + { + if (parse_integer_signed(first, last, value_integer)) + { + return token_type::value_integer; + } + } + + return token_type::uninitialized; + } + + /*! + @brief check whether Clinger's fast path can still succeed for this token + + parse_float_fast() needs a significand below 2^53. A mantissa with 17 or + more significant digits is at least 10^16 and therefore always exceeds it, + so calling the fast path would walk the token one extra time only to + decline before strtod has to run anyway. + + Significant digits are the mantissa's digits from the first nonzero one on; + the sign, the decimal point, leading zeros, and the exponent do not count. + The answer is derived from indices - the digits are not scanned again - so + this stays off the hot path of the number scanners. + + @param[in] mantissa_end offset just past the last mantissa byte in + token_buffer + @return false if parse_float_fast() is guaranteed to decline + */ + bool mantissa_fits_clinger(std::size_t mantissa_end) const + { + // 10^16 already exceeds 2^53, so 17 digits can never fit + constexpr std::size_t limit = 17; + + const std::size_t neg = (!token_buffer.empty() && token_buffer[0] == '-') ? 1u : 0u; + const std::size_t has_dot = (decimal_point_position != std::string::npos) ? 1u : 0u; + // the JSON grammar restricts the integer part to "0" or [1-9][0-9]*, so + // a leading zero can only be a lone "0", which is not significant + const std::size_t lead_zero = (token_buffer[neg] == '0') ? 1u : 0u; + JSON_ASSERT(mantissa_end >= neg + has_dot + lead_zero); + std::size_t digits = mantissa_end - neg - has_dot - lead_zero; + + if (JSON_HEDLEY_LIKELY(digits < limit)) + { + return true; + } + + // Only a number below 1 can carry further insignificant zeros, and only + // while the count stays at the limit does removing them change the + // answer - so this loop is skipped for all but a few tokens. Note + // token_buffer holds the locale's decimal point, so the fraction is + // located through decimal_point_position rather than by searching '.'. + if (lead_zero != 0) + { + JSON_ASSERT(has_dot != 0); // an integer "0" cannot reach the limit + for (std::size_t i = decimal_point_position + 1; + digits >= limit && i < mantissa_end && token_buffer[i] == '0'; ++i) + { + --digits; + } + } + + return digits < limit; + } + + /*! + @brief convert the number text in token_buffer to its value and token type + + The digit sequence in token_buffer has already been validated (by the + scan_number() state machine or by the contiguous fast path) and holds the + locale decimal point in place of '.'. Integers are parsed first and fall + back to floating point on overflow. This is shared so both scanners produce + identical results. + + @param[in] mantissa_end offset just past the last mantissa byte in + token_buffer (the index of 'e'/'E', or + token_buffer.size() when there is no exponent); + used to skip Clinger's fast path when it cannot + possibly succeed - see mantissa_fits_clinger() + */ + token_type convert_number(token_type number_type, std::size_t mantissa_end) + { // If the caller does not need the converted value (only whether the // input is syntactically valid; see json_sax_acceptor/accept()), an // unsigned/integer token can be reported without calling @@ -9129,45 +9972,37 @@ scan_number_done: } } - char* endptr = nullptr; // NOLINT(misc-const-correctness,cppcoreguidelines-pro-type-vararg,hicpp-vararg) - errno = 0; + const char* const num_begin = token_buffer.data(); + const char* const num_end = num_begin + token_buffer.size(); - // try to parse integers first and fall back to floats - if (number_type == token_type::value_unsigned) + if (number_type != token_type::value_float) { - const auto x = std::strtoull(token_buffer.data(), &endptr, 10); - - // we checked the number format before - JSON_ASSERT(endptr == token_buffer.data() + token_buffer.size()); - - if (errno != ERANGE) + const token_type integer_result = convert_integer(number_type, num_begin, num_end); + if (integer_result != token_type::uninitialized) { - value_unsigned = static_cast(x); - if (value_unsigned == x) - { - return token_type::value_unsigned; - } - } - } - else if (number_type == token_type::value_integer) - { - const auto x = std::strtoll(token_buffer.data(), &endptr, 10); - - // we checked the number format before - JSON_ASSERT(endptr == token_buffer.data() + token_buffer.size()); - - if (errno != ERANGE) - { - value_integer = static_cast(x); - if (value_integer == x) - { - return token_type::value_integer; - } + return integer_result; } } // this code is reached if we parse a floating-point number or if an - // integer conversion above failed + // integer conversion above overflowed. Prefer std::from_chars + // (Eisel-Lemire, locale-independent, correctly rounded) when available; + // otherwise the exact Clinger fast path (double only); otherwise the + // locale-aware strtof/strtod. + if (parse_float_from_chars(num_begin, num_end, value_float)) + { + return token_type::value_float; + } + // Skipping a fast path that cannot succeed is lossless and saves a full + // extra pass over the token's bytes, which otherwise shows up on + // high-precision inputs such as canada.json + if (mantissa_fits_clinger(mantissa_end) + && parse_float_fast(num_begin, num_end, decimal_point_char, value_float)) + { + return token_type::value_float; + } + + char* endptr = nullptr; // NOLINT(misc-const-correctness,cppcoreguidelines-pro-type-vararg,hicpp-vararg) strtof(value_float, token_buffer.data(), &endptr); // we checked the number format before @@ -9176,6 +10011,158 @@ scan_number_done: return token_type::value_float; } + /*! + @brief contiguous fast path for scanning a number + + Parses the whole number token straight from the input buffer, avoiding the + per-character get()/add() of scan_number(). On success it fills token_buffer + (with the locale decimal point substituted, as scan_number() does) and + returns the token type. On anything it does not fully recognize as a + well-formed number it makes no state change and returns + token_type::uninitialized, so the caller falls back to scan_number(), which + then produces the exact diagnostic. @a current is the first digit or the + leading minus (already read); the remaining bytes are taken from the adapter. + */ + token_type scan_number_bulk_contiguous() + { + // a pending unget offsets the buffer position from current; fall back + if (next_unget) + { + return token_type::uninitialized; + } + const std::size_t rem = ia.bulk_remaining(); + if (rem == 0) + { + // the first digit is the last input byte; let scan_number() finish + return token_type::uninitialized; + } + // the byte before the next unread one is current (contiguous input) + const char* const data = reinterpret_cast(ia.bulk_data()) - 1; + const std::size_t avail = rem + 1; + + // validate + classify the number extent (mirrors scan_number()'s grammar) + std::size_t i = 0; + std::size_t dot_index = std::string::npos; + token_type number_type = token_type::value_unsigned; + if (data[0] == '-') + { + number_type = token_type::value_integer; + i = 1; + if (i >= avail) + { + return token_type::uninitialized; + } + } + if (data[i] == '0') + { + ++i; + } + else if (data[i] >= '1' && data[i] <= '9') + { + ++i; + while (i < avail && data[i] >= '0' && data[i] <= '9') + { + ++i; + } + } + else + { + return token_type::uninitialized; + } + if (i < avail && data[i] == '.') + { + number_type = token_type::value_float; + dot_index = i; + ++i; + if (i >= avail || !(data[i] >= '0' && data[i] <= '9')) + { + return token_type::uninitialized; + } + while (i < avail && data[i] >= '0' && data[i] <= '9') + { + ++i; + } + } + // the mantissa ends here, whether or not an exponent part follows + const std::size_t mantissa_end = i; + if (i < avail && (data[i] == 'e' || data[i] == 'E')) + { + number_type = token_type::value_float; + ++i; + if (i < avail && (data[i] == '+' || data[i] == '-')) + { + ++i; + } + if (i >= avail || !(data[i] >= '0' && data[i] <= '9')) + { + return token_type::uninitialized; + } + while (i < avail && data[i] >= '0' && data[i] <= '9') + { + ++i; + } + } + const std::size_t len = i; + + // reset() records where this token starts (for diagnostics), so it has + // to run before the input position advances below + reset(); + + // An integer token needs no token_buffer: the SAX callbacks for + // number_integer/number_unsigned take only the value, and the overflow + // diagnostic rebuilds the text from the input. Convert straight from the + // input buffer and leave token_buffer empty. (JSON_DIAGNOSTIC_POSITIONS + // derives a number's start position from get_string().size(), so there + // the token still has to be materialized.) +#if !JSON_DIAGNOSTIC_POSITIONS + if (number_type != token_type::value_float) + { + const token_type integer_result = convert_integer(number_type, data, data + len); + if (JSON_HEDLEY_LIKELY(integer_result != token_type::uninitialized)) + { + ia.bulk_skip(len - 1); + position.chars_read_total += (len - 1); + position.chars_read_current_line += (len - 1); + return integer_result; + } + // The value does not fit an integer, so this token converts as a + // float. Recording that here keeps convert_number() below from + // repeating the integer attempt that just failed. + number_type = token_type::value_float; + } +#endif + + // materialize the token exactly as scan_number() would, substituting the + // locale decimal point so convert_number()'s strtof fallback stays valid. + // reset() already cleared token_buffer, so append() fills it (assign() is + // avoided because custom string_t types need not provide it) + token_buffer.append(reinterpret_cast(data), len); + if (dot_index != std::string::npos) + { + token_buffer[dot_index] = static_cast(decimal_point_char); + decimal_point_position = dot_index; + } + + ia.bulk_skip(len - 1); + position.chars_read_total += (len - 1); + position.chars_read_current_line += (len - 1); + + return convert_number(number_type, mantissa_end); + } + + /// contiguous input: try the number fast path, else the byte-path scanner + token_type scan_number_dispatch(std::true_type /*bulk*/) + { + const token_type t = scan_number_bulk_contiguous(); + return (t != token_type::uninitialized) ? t : scan_number(); + } + + /// streaming input: always use the byte-path scanner + token_type scan_number_dispatch(std::false_type /*bulk*/) + { + return scan_number(); + } + /*! @param[in] literal_text the literal text to expect @param[in] length the length of the passed literal text @@ -9279,6 +10266,9 @@ scan_number_done: if (current == '\n') { ++position.lines_read; + // remember the column the newline was read at: chars_read_current_line + // is about to be cleared, and a matching unget() cannot reconstruct it + chars_read_before_newline = position.chars_read_current_line; position.chars_read_current_line = 0; } @@ -9335,12 +10325,20 @@ scan_number_done: --position.chars_read_total; // in case we "unget" a newline, we have to also decrement the lines_read + // and restore the column that get() cleared when it saw the newline; + // chars_read_current_line == 0 can only mean the last get() read one if (position.chars_read_current_line == 0) { if (position.lines_read > 0) { --position.lines_read; } + + // chars_read_before_newline counts the newline itself, which is the + // character being ungotten, hence the -1 + position.chars_read_current_line = (chars_read_before_newline > 0) + ? chars_read_before_newline - 1 + : 0; } else { @@ -9607,7 +10605,7 @@ scan_number_done: case '7': case '8': case '9': - return scan_number(); + return scan_number_dispatch(std::integral_constant {}); // end of input (the null byte is needed when parsing from // string literals) @@ -9638,6 +10636,10 @@ scan_number_done: /// the start position of the current token position_t position {}; + /// the value chars_read_current_line had when the last newline was read, so + /// that unget() can restore the column instead of leaving it at 0 + std::size_t chars_read_before_newline = 0; + /// raw input token string for error messages; only populated for streaming /// adapters (seekable adapters reconstruct it lazily via token_string_start) std::vector token_string {}; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 2d0aaaf70..322e40d80 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -2,6 +2,9 @@ cmake_minimum_required(VERSION 3.13...4.0) option(JSON_Valgrind "Execute test suite with Valgrind." OFF) option(JSON_FastTests "Skip expensive/slow tests." OFF) +option(JSON_TestSimdutf "Build the unit tests against the simdutf UTF-8 validation backend." OFF) + +set(JSON_SIMDUTF_VERSION 9.1.0 CACHE STRING "The simdutf version used by JSON_TestSimdutf.") set(JSON_32bitTest AUTO CACHE STRING "Enable the 32bit unit test (ON/OFF/AUTO/ONLY).") set(JSON_TestStandards "" CACHE STRING "The list of standards to test explicitly.") @@ -194,6 +197,71 @@ if(test_force) endif() message(STATUS "${msg}") +############################################################################# +# optionally validate UTF-8 with simdutf (JSON_USE_SIMDUTF) +############################################################################# + +# The simdutf backend is opt-in and not vendored, so it is fetched here rather +# than being a checked-in dependency. Everything below hangs off test_main, +# whose usage requirements every test target inherits; the library target and +# the installed CMake package are deliberately left untouched. +if (JSON_TestSimdutf) + # simdutf requires C++17, both to compile itself and to be reachable from + # the library, which keeps its scalar validator below that. Find a tested + # standard that satisfies it. + set(simdutf_standard "") + foreach(cxx_standard ${test_cxx_standards}) + if(NOT cxx_standard LESS 17 AND compiler_supports_cpp_${cxx_standard}) + set(simdutf_standard ${cxx_standard}) + break() + endif() + endforeach() + + if("${simdutf_standard}" STREQUAL "") + # Building simdutf would fail outright without a C++17 compiler, and + # even with one it would go unused if no C++17-or-later standard is + # tested. Say so and fall back to the scalar validator rather than + # failing the build. + if(NOT compiler_supports_cpp_17) + set(simdutf_reason "the compiler does not support C++17") + else() + set(simdutf_reason "no tested standard is C++17 or later (testing ${msg_standards})") + endif() + message(WARNING + "JSON_TestSimdutf is enabled, but ${simdutf_reason}. simdutf requires C++17, so it " + "is not fetched and JSON_USE_SIMDUTF is not defined: the tests run against the " + "built-in scalar UTF-8 validator instead. Set JSON_TestStandards to include 17 or " + "later, or build with a compiler that supports C++17.") + else() + if (CMAKE_VERSION VERSION_LESS 3.18) + message(FATAL_ERROR "JSON_TestSimdutf requires CMake 3.18 or later (simdutf's minimum).") + endif() + + include(FetchContent) + + # simdutf builds its tests and tools by default, and its tests pull + # further dependencies of their own; only the library is needed here + set(SIMDUTF_TESTS OFF CACHE BOOL "" FORCE) + set(SIMDUTF_TOOLS OFF CACHE BOOL "" FORCE) + set(SIMDUTF_BENCHMARKS OFF CACHE BOOL "" FORCE) + set(SIMDUTF_ICONV OFF CACHE BOOL "" FORCE) + + FetchContent_Declare(simdutf + URL https://github.com/simdutf/simdutf/archive/refs/tags/v${JSON_SIMDUTF_VERSION}.tar.gz + DOWNLOAD_EXTRACT_TIMESTAMP TRUE + ) + FetchContent_MakeAvailable(simdutf) + + target_compile_definitions(test_main PUBLIC JSON_USE_SIMDUTF) + target_link_libraries(test_main PUBLIC simdutf::simdutf) + + # simdutf.h requires C++17; below that the library keeps its scalar + # validator, so any C++11/14 test targets exercise the fallback and the + # C++17-and-later ones exercise simdutf. Both must agree. + message(STATUS "UTF-8 validation delegated to simdutf ${JSON_SIMDUTF_VERSION} for C++17 and later (JSON_USE_SIMDUTF)") + endif() +endif() + # *DO* use json_test_set_test_options() above this line json_test_should_build_32bit_test(json_32bit_test json_32bit_test_only "${JSON_32bitTest}") diff --git a/tests/src/unit-class_lexer.cpp b/tests/src/unit-class_lexer.cpp index 64baf3da6..e89497738 100644 --- a/tests/src/unit-class_lexer.cpp +++ b/tests/src/unit-class_lexer.cpp @@ -12,6 +12,11 @@ #include using nlohmann::json; +#include // strtod +#include // stringstream +#include // string +#include // vector + namespace { // shortcut to scan a string literal @@ -224,3 +229,431 @@ TEST_CASE("lexer class") CHECK((scan_string("/**//**//**/", true) == json::lexer::token_type::end_of_input)); } } + +TEST_CASE("lexer number fast path") +{ + // The contiguous fast path (used for pointer/string input) must agree with + // the streaming byte path (used for std::istream) on token type, numeric + // value, and round-trip text for every well-formed number, and reject the + // same malformed numbers with the same message. + SECTION("contiguous vs streaming parity") + { + const std::vector numbers = + { + "0", "-0", "1", "-1", "42", "-42", "10", "100", "1234567890", + "0.0", "-0.0", "3.14", "-3.14", "0.5", "-0.001", "123.456789", + "1e0", "1E0", "1e10", "1e-10", "1e+10", "1.5e3", "-2.5E-4", + "9223372036854775807", // INT64_MAX -> unsigned + "9223372036854775808", // INT64_MAX + 1 -> unsigned + "18446744073709551615", // UINT64_MAX -> unsigned + "18446744073709551616", // UINT64_MAX + 1 -> float + "-9223372036854775808", // INT64_MIN -> integer + "-9223372036854775809", // INT64_MIN - 1 -> float + "123456789012345678901234567890", // huge -> float + "0.30000000000000004", "2.2250738585072014e-308", "1e308", + // high-precision / wide-exponent values that exercise the + // std::from_chars (Eisel-Lemire) path beyond the Clinger subset + "1.7976931348623157e308", "1.2345678901234567e-250", + "9007199254740993", "5e-324", "1e-320" + }; + + for (const auto& n : numbers) + { + const std::string doc = "[" + n + "]"; + + // contiguous fast path + const json a = json::parse(doc); + // streaming byte path + std::stringstream ss(doc); + const json b = json::parse(ss); + + CAPTURE(n); + CHECK(a == b); + CHECK(a.dump() == b.dump()); + CHECK(a[0].type() == b[0].type()); + } + } + + SECTION("significant-digit gate for the Clinger fast path") + { + // Clinger's fast path needs a significand below 2^53, so it cannot + // succeed once the mantissa has 17 or more significant digits (the + // significand would be at least 10^16). The lexer skips the attempt + // there. That is only allowed to save work: every value must still come + // out bit-exactly, and both scanners must agree. In particular the gate + // must not fire for tokens whose leading zeros merely look like extra + // digits - "0.1234567890123456" has 16 significant digits, not 17. + const std::vector numbers = + { + "1234567890123456", // 16 significant digits + "12345678901234567", // 17 -> attempt skipped + "123456789012345678", // 18 -> attempt skipped + "0.1234567890123456", // 16: the leading "0" is not significant + "0.12345678901234567", // 17 + "0.00000000000000001", // 1, in a long token + "0.000000000000000012345678901234", // 14, in a long token + "-0.0000000000000000000001", // 1, negative + "1.0000000000000000", // 17: trailing zeros are significant here + "10000000000000000", // 17 + "9007199254740992", // 2^53 + "9007199254740993", // 2^53 + 1 + "-65.613616999999977", // canada.json shape + "1.2345678901234567e-250", // 17 with an exponent + "1.234567890123456e-250", // 16 with an exponent + "1e10", "0.0", "-0.0", "0e0", "0.000123" + }; + + for (const auto& n : numbers) + { + CAPTURE(n); + const std::string doc = "[" + n + "]"; + + const json a = json::parse(doc); // contiguous fast path + std::stringstream ss(doc); + const json b = json::parse(ss); // streaming byte path + + CHECK(a[0].type() == b[0].type()); + CHECK(a == b); + + if (a[0].is_number_float()) + { + const double expected = std::strtod(n.c_str(), nullptr); + CHECK(a[0].get() == expected); + CHECK(b[0].get() == expected); + } + } + } + + SECTION("token type classification") + { + CHECK((scan_string("0") == json::lexer::token_type::value_unsigned)); + CHECK((scan_string("-1") == json::lexer::token_type::value_integer)); + CHECK((scan_string("1.5") == json::lexer::token_type::value_float)); + CHECK((scan_string("1e5") == json::lexer::token_type::value_float)); + CHECK((scan_string("18446744073709551615") == json::lexer::token_type::value_unsigned)); + CHECK((scan_string("18446744073709551616") == json::lexer::token_type::value_float)); + CHECK((scan_string("-9223372036854775808") == json::lexer::token_type::value_integer)); + CHECK((scan_string("-9223372036854775809") == json::lexer::token_type::value_float)); + } + + SECTION("malformed numbers are rejected identically") + { + for (const char* bad : + {"-", "1.", "1e", "1e+", "1.2e", "01", "-01", "1..2", "1.2.3" + }) + { + CAPTURE(bad); + // the contiguous fast path must decline and let the byte path report + const std::string doc = std::string("[") + bad + "]"; + CHECK_FALSE(json::accept(doc)); + std::stringstream ss(doc); + CHECK_FALSE(json::accept(ss)); + } + } + +#if !defined(JSON_NOEXCEPTION) + // these sections parse invalid input, which aborts when exceptions are off + SECTION("exhaustive grammar parity with the streaming path") + { + // The JSON number grammar is encoded twice: once as the scan_number() + // state machine and once as the contiguous fast path. Enumerate every + // short string over the number alphabet and require the two encodings to + // agree exactly - on acceptance, on the reported error, and on the parsed + // value - so they cannot drift apart. + const std::string alphabet = "01.eE+-"; + + // full outcome of parsing @a doc, so a mismatch in type, value, or error + // message is caught, not just a mismatch in acceptance + const auto outcome = [](const std::string & doc, bool streaming) -> std::string + { + try + { + if (streaming) + { + std::stringstream ss(doc); + const json j = json::parse(ss); + return std::string(j[0].type_name()) + '|' + j.dump(); + } + const json j = json::parse(doc); + return std::string(j[0].type_name()) + '|' + j.dump(); + } + catch (const json::parse_error& e) + { + return {e.what()}; + } + }; + + std::vector mismatches; + std::vector tokens{""}; + for (std::size_t length = 1; length <= 4; ++length) + { + std::vector next; + next.reserve(tokens.size() * alphabet.size()); + for (const auto& prefix : tokens) + { + for (const char c : alphabet) + { + next.push_back(prefix + c); + } + } + tokens = next; + + for (const auto& token : tokens) + { + const std::string doc = "[" + token + "]"; + if (outcome(doc, false) != outcome(doc, true)) + { + mismatches.push_back(doc); + } + } + } + + // 7 + 49 + 343 + 2401 tokens + CHECK(tokens.size() == 2401); + CAPTURE(mismatches); + CHECK(mismatches.empty()); + } + + SECTION("error positions match the streaming path") + { + // Rejecting identically is not enough: the fast path must also report the + // error at the same position as the byte path. A number directly followed + // by a newline is the interesting case, because the byte path reaches the + // newline (which resets the column) and then ungets it. + // returns the parse_error message, or "" if the document parsed + const auto contiguous_error = [](const std::string & doc) -> std::string + { + try + { + const json j = json::parse(doc); + static_cast(j); + } + catch (const json::parse_error& e) + { + return {e.what()}; + } + return {}; + }; + const auto streaming_error = [](const std::string & doc) -> std::string + { + try + { + std::stringstream ss(doc); + const json j = json::parse(ss); + static_cast(j); + } + catch (const json::parse_error& e) + { + return {e.what()}; + } + return {}; + }; + + for (const char* bad : + {"[01\n]", "[00\n]", "[-01\n]", "{1\n}", "[1\n2]", "[1.2.3\n]", + "[1 \n2]", "[\n1\n2]", "1\n2", "[01\r\n]", "[1e\n]", "[-\n]" + }) + { + CAPTURE(bad); + const std::string doc = bad; + const std::string contiguous_what = contiguous_error(doc); + + CHECK_FALSE(contiguous_what.empty()); + CHECK(contiguous_what == streaming_error(doc)); + } + + // A number terminated by a newline must report the same position as the + // same number terminated by anything else: scan_number() reads the + // terminator and ungets it, so the reported column is the one reached + // after the number's last character - not the 0 that an unget() across + // the newline used to leave behind. + CHECK(contiguous_error("[01\n]") == contiguous_error("[01 ]")); + CHECK(contiguous_error("[01\n]") == + "[json.exception.parse_error.101] parse error at line 1, column 3: " + "syntax error while parsing array - unexpected number literal; expected ']'"); + + // the same for a multi-character token, where the column of the last + // character (the '3' of "-2.5e3") differs from the column it starts at + CHECK(contiguous_error("null -2.5e3\nfalse") == contiguous_error("null -2.5e3 false")); + CHECK(contiguous_error("null -2.5e3\nfalse") == + "[json.exception.parse_error.101] parse error at line 1, column 11: " + "syntax error while parsing value - unexpected number literal; expected end of input"); + } +#endif +} + +TEST_CASE("lexer string fast path") +{ + // Build a byte string from explicit values: a hex escape in a string + // literal swallows every following hex digit, which makes sequences like + // "\xC3\xA9b" mean something other than they look like. + const auto bytes = [](std::initializer_list values) + { + std::string result; + for (const int value : values) + { + result.push_back(static_cast(value)); + } + return result; + }; + +#if !defined(JSON_NOEXCEPTION) + // the full outcome of parsing @a doc: the parsed value, or the exact error + // message, so a mismatch in either is caught. Only usable with exceptions + // on: parsing invalid input aborts when they are off. + const auto outcome = [](const std::string & doc, bool streaming) -> std::string + { + try + { + if (streaming) + { + std::stringstream ss(doc); + const json j = json::parse(ss); + return j.dump(); + } + const json j = json::parse(doc); + return j.dump(); + } + // not just parse_error: if a bulk scanner ever let ill-formed UTF-8 + // through, dump() would throw type_error.316, and that has to surface + // as a reported mismatch rather than as an uncaught exception + catch (const json::exception& e) + { + return {e.what()}; + } + }; +#endif + + // once at the start of the string, once past the first 8-byte SWAR word, so + // the bulk scanner sees each case with and without a run behind it + const std::vector offsets{0, 9}; + +#if !defined(JSON_NOEXCEPTION) + SECTION("exhaustive contiguous vs streaming parity") + { + // ordinary ASCII, both specials, a control byte, characters that make + // the preceding backslash a valid escape, a UTF-8 lead byte of each + // length, a continuation byte, and a byte that is never valid + const std::vector alphabet = + { + "a", "\"", "\\", "n", "u", "0", bytes({0x01}), + bytes({0xC3}), bytes({0xA9}), bytes({0xE4}), bytes({0xF0}), + bytes({0x80}), bytes({0xFF}) + }; + + std::vector mismatches; + std::vector tokens{""}; + for (std::size_t length = 1; length <= 3; ++length) + { + std::vector next; + next.reserve(tokens.size() * alphabet.size()); + for (const auto& prefix : tokens) + { + for (const auto& symbol : alphabet) + { + next.push_back(prefix + symbol); + } + } + tokens = next; + + for (const auto& token : tokens) + { + for (const std::size_t offset : offsets) + { + const std::string doc = "[\"" + std::string(offset, 'a') + token + "\"]"; + if (outcome(doc, false) != outcome(doc, true)) + { + mismatches.push_back(doc); + } + } + } + } + + // 13 + 169 + 2197 tokens, each at two offsets + CHECK(tokens.size() == 2197); + CAPTURE(mismatches); + CHECK(mismatches.empty()); + } + + SECTION("special bytes at every offset of the SWAR stride") + { + // The bulk scanner consumes 8 bytes at a time and then a tail; place + // every kind of byte that ends a run at each offset across two words, + // so multibyte sequences also straddle the word boundary. + const std::vector specials = + { + "\"", "\\", bytes({0x01}), bytes({0x1F}), bytes({0x7F}), + bytes({0xC3, 0xA9}), bytes({0xE4, 0xB8, 0xAD}), bytes({0xF0, 0x9F, 0x98, 0x80}), + bytes({0xFF}), bytes({0xC3}), bytes({0xE4, 0xB8}) + }; + + std::vector mismatches; + for (std::size_t offset = 0; offset <= 17; ++offset) + { + for (const auto& special : specials) + { + const std::string doc = "[\"" + std::string(offset, 'a') + special + "\"]"; + if (outcome(doc, false) != outcome(doc, true)) + { + mismatches.push_back(doc); + } + } + } + CAPTURE(mismatches); + CHECK(mismatches.empty()); + } +#endif + + // json::accept() never throws, so the ranges stay covered without exceptions + SECTION("UTF-8 ranges are accepted and rejected as documented") + { + // The bulk validator must accept exactly what the byte-at-a-time + // scanner accepts, so pin the boundaries of every range it recognizes. + // aggregate, only ever brace-initialized below; default member + // initializers would stop it being an aggregate in C++11 + struct utf8_case // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init) + { + std::string sequence; + bool valid; + const char* description; + }; + const std::vector cases = + { + {bytes({0xC2, 0x80}), true, "U+0080, shortest two-byte"}, + {bytes({0xDF, 0xBF}), true, "U+07FF, longest two-byte"}, + {bytes({0xC1, 0xBF}), false, "overlong two-byte"}, + {bytes({0xC2, 0x7F}), false, "two-byte with bad continuation"}, + {bytes({0xE0, 0xA0, 0x80}), true, "U+0800, shortest three-byte"}, + {bytes({0xE0, 0x9F, 0xBF}), false, "overlong three-byte"}, + {bytes({0xED, 0x9F, 0xBF}), true, "U+D7FF, just below the surrogates"}, + {bytes({0xED, 0xA0, 0x80}), false, "surrogate U+D800"}, + {bytes({0xED, 0xBF, 0xBF}), false, "surrogate U+DFFF"}, + {bytes({0xEE, 0x80, 0x80}), true, "U+E000, just above the surrogates"}, + {bytes({0xEF, 0xBF, 0xBF}), true, "U+FFFF"}, + {bytes({0xF0, 0x90, 0x80, 0x80}), true, "U+10000, shortest four-byte"}, + {bytes({0xF0, 0x8F, 0xBF, 0xBF}), false, "overlong four-byte"}, + {bytes({0xF4, 0x8F, 0xBF, 0xBF}), true, "U+10FFFF, highest code point"}, + {bytes({0xF4, 0x90, 0x80, 0x80}), false, "above U+10FFFF"}, + {bytes({0xF5, 0x80, 0x80, 0x80}), false, "lead byte out of range"}, + {bytes({0x80}), false, "bare continuation byte"}, + {bytes({0xFF}), false, "byte that never appears in UTF-8"}, + {bytes({0xC3}), false, "truncated two-byte"}, + {bytes({0xE4, 0xB8}), false, "truncated three-byte"}, + {bytes({0xF0, 0x9F, 0x98}), false, "truncated four-byte"} + }; + + for (const auto& test_case : cases) + { + CAPTURE(test_case.description); + for (const std::size_t offset : offsets) + { + CAPTURE(offset); + const std::string doc = "[\"" + std::string(offset, 'a') + test_case.sequence + "\"]"; + CHECK(json::accept(doc) == test_case.valid); +#if !defined(JSON_NOEXCEPTION) + CHECK(outcome(doc, false) == outcome(doc, true)); +#endif + } + } + } +} diff --git a/tests/src/unit-user_defined_input.cpp b/tests/src/unit-user_defined_input.cpp index 823e82862..f07a8a608 100644 --- a/tests/src/unit-user_defined_input.cpp +++ b/tests/src/unit-user_defined_input.cpp @@ -18,7 +18,12 @@ #include using nlohmann::json; +#include // array +#include // size_t +#include // uint8_t #include +#include // string +#include // vector #if defined(__cpp_lib_concepts) && defined(JSON_HAS_CPP_20) #include @@ -212,6 +217,66 @@ TEST_CASE("Parse with heterogeneous iterator and sentinel types") CHECK(j2.at(0) == 1); } +// A type whose data() hands out raw bytes but whose size() counts something +// else - here fixed-size records. Reading [data(), data() + size()) as bytes +// would silently truncate the input, so data() and size() alone must not be +// taken as evidence of contiguous byte storage. +struct record_buffer +{ + using value_type = std::array; + + std::string bytes; + + const char* data() const noexcept + { + return bytes.data(); + } + std::size_t size() const noexcept + { + return bytes.size() / sizeof(value_type); + } + const char* begin() const noexcept + { + return bytes.data(); + } + const char* end() const noexcept + { + return bytes.data() + bytes.size(); + } +}; + +TEST_CASE("Contiguous byte containers take the pointer adapter") +{ + // Containers with contiguous single-byte storage are routed through the + // pointer-based adapter so the bulk fast paths apply in every standard, not + // only in C++20 where the library iterators model std::contiguous_iterator. + CHECK(nlohmann::detail::is_contiguous_byte_container::value); + CHECK(nlohmann::detail::is_contiguous_byte_container>::value); + CHECK(nlohmann::detail::is_contiguous_byte_container>::value); + CHECK(nlohmann::detail::is_contiguous_byte_container>::value); + + // input_adapter() takes its container by forwarding reference, so the trait + // is also asked about reference types + CHECK(nlohmann::detail::is_contiguous_byte_container::value); + CHECK(nlohmann::detail::is_contiguous_byte_container::value); + + // everything else keeps the iterator-based adapter + CHECK_FALSE(nlohmann::detail::is_contiguous_byte_container>::value); + CHECK_FALSE(nlohmann::detail::is_contiguous_byte_container>::value); + CHECK_FALSE(nlohmann::detail::is_contiguous_byte_container::value); + + // including a type that has data() and size() but whose size() does not + // count the units data() points at: its value_type says so + CHECK_FALSE(nlohmann::detail::is_contiguous_byte_container::value); + + // and such a container still parses through its iterators, in full - taking + // it for a byte container would stop after data() + size() bytes + const record_buffer buffer{"[1,2,3,4,5]"}; + CHECK(buffer.data() == buffer.bytes.data()); + CHECK(buffer.size() * sizeof(record_buffer::value_type) < buffer.bytes.size()); + CHECK(json::parse(buffer) == json({1, 2, 3, 4, 5})); +} + #if defined(__cpp_lib_concepts) && defined(JSON_HAS_CPP_20) // JSON_HAS_CPP_20 (do not remove; see note at top of file) TEST_CASE("Parse with std::counted_iterator and std::default_sentinel_t") @@ -228,6 +293,180 @@ TEST_CASE("Parse with std::counted_iterator and std::default_sentinel_t") const std::counted_iterator first2(json_str.begin(), len); CHECK(json::accept(first2, std::default_sentinel)); } + +TEST_CASE("std::counted_iterator reaches the contiguous fast paths") +{ + // A sized sentinel makes the remaining element count computable in O(1), so + // std::counted_iterator over a contiguous iterator must reach the same bulk + // string/number scanners as a plain pointer - not just the byte-at-a-time + // fallback (see #5268 for the equivalent memcpy fast path). +#if JSON_HAS_RANGES + // JSON_HAS_RANGES is 0 on standard libraries with an incomplete + // (libstdc++ < 11, libc++ < 16), where the adapter deliberately falls back + // to the byte-at-a-time scanner; everything below still has to work there. + using adapter_type = nlohmann::detail::iterator_input_adapter, std::default_sentinel_t>; + CHECK(adapter_type::supports_bulk_scan); + CHECK(adapter_type::supports_seek); +#endif + + // exercise every fast path: long ASCII run, multibyte UTF-8, escapes, and + // integer/floating-point numbers + const std::string json_str = + R"({"ascii":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",)" + "\"utf8\":\"\xe4\xb8\xad\xe6\x96\x87\xf0\x9f\x98\x80\xc3\xa9\"," + R"("escaped":"aéb\n\\","ints":[0,-1,18446744073709551615,-9223372036854775808],)" + R"("floats":[1.5,-2.25e3,0.30000000000000004]})"; + const auto len = static_cast>(json_str.size()); + + const std::counted_iterator first(json_str.data(), len); + const json j = json::parse(first, std::default_sentinel); + + // parsing through the pointer adapter must give exactly the same result + CHECK(j == json::parse(json_str)); + +#if !defined(JSON_NOEXCEPTION) + // Diagnostics that quote the offending token are reconstructed from the + // already-consumed input (supports_seek), a path a sized sentinel only + // reaches now; check a few that include the "last read" text. Parsing + // invalid input aborts when exceptions are off, hence the guard. + // Raw strings and explicit bytes: an escaped literal and two literals + // written next to each other both read as mistakes to static analysis. + const auto byte = [](int value) + { + return std::string(1, static_cast(value)); + }; + const std::vector diagnostic_docs = + { + "1\nx", + "truX", + "[tru]", + R"("abc)", + R"(["\ud834"])", + R"(["a)" + byte(0x01) + R"(b"])", + R"([")" + byte(0xC3) + byte(0x28) + R"("])", + "[1e]", + R"(["aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaX)" + }; + + for (const auto& text : diagnostic_docs) + { + CAPTURE(text); + const std::counted_iterator it(text.data(), static_cast>(text.size())); + std::string counted_message; + std::string string_message; + try + { + const json counted_result = json::parse(it, std::default_sentinel); + static_cast(counted_result); + } + catch (const json::parse_error& e) + { + counted_message = e.what(); + } + try + { + const json string_result = json::parse(text); + static_cast(string_result); + } + catch (const json::parse_error& e) + { + string_message = e.what(); + } + CHECK_FALSE(counted_message.empty()); + CHECK(counted_message == string_message); + } + + // and errors must still be reported identically + const std::string bad = "[01\n]"; + const std::counted_iterator bad_first(bad.data(), static_cast>(bad.size())); + std::string counted_what; + std::string string_what; + try + { + const json counted_result = json::parse(bad_first, std::default_sentinel); + static_cast(counted_result); + } + catch (const json::parse_error& e) + { + counted_what = e.what(); + } + try + { + const json string_result = json::parse(bad); + static_cast(string_result); + } + catch (const json::parse_error& e) + { + string_what = e.what(); + } + CHECK_FALSE(counted_what.empty()); + CHECK(counted_what == string_what); +#endif +} + +#if !defined(JSON_NOEXCEPTION) +// several cases below are truncated on purpose, and parsing invalid input +// aborts when exceptions are off +TEST_CASE("std::counted_iterator bulk scanning stops at the counted end") +{ + // The count, not the size of the underlying buffer, is the end of the + // input: the bulk scanners must never look at the bytes behind it, even + // though they are readable. Each case is compared against parsing the + // equivalent prefix as a std::string. + const auto via_counted = [](const std::string & buf, std::size_t n) -> std::string + { + const std::counted_iterator first(buf.data(), static_cast>(n)); + try + { + const json j = json::parse(first, std::default_sentinel); + return "OK|" + j.dump(); + } + catch (const json::parse_error& e) + { + return {e.what()}; + } + }; + const auto via_prefix = [](const std::string & buf, std::size_t n) -> std::string + { + try + { + const json j = json::parse(buf.substr(0, n)); + return "OK|" + j.dump(); + } + catch (const json::parse_error& e) + { + return {e.what()}; + } + }; + + struct testcase // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init) + { + const char* buffer; + std::size_t count; + }; + const std::vector cases = + { + {"[\"abc\"]____TRAILING____", 7}, // exact fit, tail hidden + {"[\"abcdefghijklmnop\"]____", 8}, // cut inside a string + {"[\"abc\"]____", 6}, // cut just before the closing quote + {"[12345]xxxxx", 4}, // cut inside a number + {"[123]999999", 5}, // number ends exactly at the count + {"[\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"]", 12}, // closing quote only behind the count + {"[\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"]", 19}, // cut inside an 8-byte SWAR stride + {"[\"\xe4\xb8\xad\xe6\x96\x87\"]", 5}, // cut inside a UTF-8 sequence + {"[\"\xe4\xb8\xad\xe6\x96\x87\"]____", 10}, // complete UTF-8, tail hidden + {"[1.25e3]TRAILINGDIGITS999", 7}, // number token reaches the count + }; + + for (const auto& tc : cases) + { + CAPTURE(tc.buffer); + CAPTURE(tc.count); + const std::string buffer = tc.buffer; + CHECK(via_counted(buffer, tc.count) == via_prefix(buffer, tc.count)); + } +} +#endif #endif } // namespace From bfb07786cd7eb841a6d9030bcd822d3b1c4d3b56 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Thu, 10 Sep 2026 17:59:28 +0200 Subject: [PATCH 04/18] Fix to_bjdata() silently truncating out-of-range _ArrayData_ elements (#5473) write_bjdata_ndarray() validated that each _ArrayData_ element matched the number kind (integer vs. float) named by _ArrayType_, but not its range. An element that did not fit the target C++ type (e.g. 256 for "uint8") was silently wrapped by the static_cast used to write it, or, for "single", silently overflowed to infinity. Range-check each element against the type named by _ArrayType_ before writing it, reusing the existing fallback path that already encodes the annotated object as a plain object for other invalid-annotation cases in this function. Fixes #5403. Signed-off-by: Niels Lohmann --- .../nlohmann/detail/output/binary_writer.hpp | 68 +++++++++++++++++++ single_include/nlohmann/json.hpp | 68 +++++++++++++++++++ tests/src/unit-bjdata.cpp | 47 +++++++++++++ 3 files changed, 183 insertions(+) diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 496c733d1..9514f0fcd 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -1647,6 +1647,20 @@ class binary_writer return 'D'; // float 64 } + /*! + @brief checks whether a JSON number fits into @a TargetType + @param[in] el a JSON number of either the signed or unsigned integer kind + @return whether @a el's value can be represented by @a TargetType without + wrapping, regardless of which of the two kinds it is stored as + */ + template + static bool bjdata_ndarray_value_in_range(const BasicJsonType& el) + { + return el.is_number_unsigned() + ? value_in_range_of(el.template get()) + : value_in_range_of(el.template get()); + } + /*! @return false if the object is successfully converted to a bjdata ndarray, true if the type or size is invalid */ @@ -1731,6 +1745,60 @@ class binary_writer } } + // every element is cast to the (possibly narrower) C++ type matching + // dtype below; a value that does not fit that type would silently + // wrap (integers) or overflow to infinity (the "single" precision + // float) instead of being reported, so such an object falls back to + // a plain object encoding as well + for (const auto& el : value.at(key)) + { + bool in_range = true; + switch (dtype) + { + case 'U': + case 'C': + case 'B': + in_range = bjdata_ndarray_value_in_range(el); + break; + case 'i': + in_range = bjdata_ndarray_value_in_range(el); + break; + case 'u': + in_range = bjdata_ndarray_value_in_range(el); + break; + case 'I': + in_range = bjdata_ndarray_value_in_range(el); + break; + case 'm': + in_range = bjdata_ndarray_value_in_range(el); + break; + case 'l': + in_range = bjdata_ndarray_value_in_range(el); + break; + case 'M': + in_range = bjdata_ndarray_value_in_range(el); + break; + case 'L': + in_range = bjdata_ndarray_value_in_range(el); + break; + case 'd': + { + const auto dval = el.template get(); + in_range = !std::isfinite(dval) || + (dval >= static_cast(std::numeric_limits::lowest()) && + dval <= static_cast((std::numeric_limits::max)())); + break; + } + default: + // 'D' (double) already spans the full range of number_float_t + break; + } + if (!in_range) + { + return true; + } + } + oa->write_character('['); oa->write_character('$'); oa->write_character(dtype); diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 10e7fff6f..34db800b7 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -19809,6 +19809,20 @@ class binary_writer return 'D'; // float 64 } + /*! + @brief checks whether a JSON number fits into @a TargetType + @param[in] el a JSON number of either the signed or unsigned integer kind + @return whether @a el's value can be represented by @a TargetType without + wrapping, regardless of which of the two kinds it is stored as + */ + template + static bool bjdata_ndarray_value_in_range(const BasicJsonType& el) + { + return el.is_number_unsigned() + ? value_in_range_of(el.template get()) + : value_in_range_of(el.template get()); + } + /*! @return false if the object is successfully converted to a bjdata ndarray, true if the type or size is invalid */ @@ -19893,6 +19907,60 @@ class binary_writer } } + // every element is cast to the (possibly narrower) C++ type matching + // dtype below; a value that does not fit that type would silently + // wrap (integers) or overflow to infinity (the "single" precision + // float) instead of being reported, so such an object falls back to + // a plain object encoding as well + for (const auto& el : value.at(key)) + { + bool in_range = true; + switch (dtype) + { + case 'U': + case 'C': + case 'B': + in_range = bjdata_ndarray_value_in_range(el); + break; + case 'i': + in_range = bjdata_ndarray_value_in_range(el); + break; + case 'u': + in_range = bjdata_ndarray_value_in_range(el); + break; + case 'I': + in_range = bjdata_ndarray_value_in_range(el); + break; + case 'm': + in_range = bjdata_ndarray_value_in_range(el); + break; + case 'l': + in_range = bjdata_ndarray_value_in_range(el); + break; + case 'M': + in_range = bjdata_ndarray_value_in_range(el); + break; + case 'L': + in_range = bjdata_ndarray_value_in_range(el); + break; + case 'd': + { + const auto dval = el.template get(); + in_range = !std::isfinite(dval) || + (dval >= static_cast(std::numeric_limits::lowest()) && + dval <= static_cast((std::numeric_limits::max)())); + break; + } + default: + // 'D' (double) already spans the full range of number_float_t + break; + } + if (!in_range) + { + return true; + } + } + oa->write_character('['); oa->write_character('$'); oa->write_character(dtype); diff --git a/tests/src/unit-bjdata.cpp b/tests/src/unit-bjdata.cpp index 7d0dd5ff2..78ddf5d8c 100644 --- a/tests/src/unit-bjdata.cpp +++ b/tests/src/unit-bjdata.cpp @@ -2776,6 +2776,53 @@ TEST_CASE("BJData") CHECK(out_num.at(0) == '{'); CHECK(json::from_bjdata(out_num) == j_num); } + + SECTION("ndarray with out-of-range _ArrayData_ elements stays as object") + { + // each element is cast to the (possibly narrower) C++ type + // named by _ArrayType_ before being written; a value that + // does not fit that type would silently wrap instead of + // being reported, so such an object falls back to a plain + // object encoding that still round-trips (see GitHub issue #5403) + + // an unsigned element that does not fit uint8 + json const j_uint8 = json({{"_ArrayType_", "uint8"}, {"_ArraySize_", {2}}, {"_ArrayData_", {1, 256}}}); + const auto out_uint8 = json::to_bjdata(j_uint8); + CHECK(out_uint8.at(0) == '{'); + CHECK(json::from_bjdata(out_uint8) == j_uint8); + + // a signed element that does not fit int8 + json const j_int8 = json({{"_ArrayType_", "int8"}, {"_ArraySize_", {2}}, {"_ArrayData_", {1, 200}}}); + const auto out_int8 = json::to_bjdata(j_int8); + CHECK(out_int8.at(0) == '{'); + CHECK(json::from_bjdata(out_int8) == j_int8); + + // a negative element is likewise out of range for an + // unsigned _ArrayType_ + json const j_uint16_neg = json({{"_ArrayType_", "uint16"}, {"_ArraySize_", {2}}, {"_ArrayData_", {1, -1}}}); + const auto out_uint16_neg = json::to_bjdata(j_uint16_neg); + CHECK(out_uint16_neg.at(0) == '{'); + CHECK(json::from_bjdata(out_uint16_neg) == j_uint16_neg); + + // a double element that overflows to infinity when narrowed + // to the "single" (float) precision named by _ArrayType_ + json const j_single = json({{"_ArrayType_", "single"}, {"_ArraySize_", {2}}, {"_ArrayData_", {1.5, 1e40}}}); + const auto out_single = json::to_bjdata(j_single); + CHECK(out_single.at(0) == '{'); + CHECK(json::from_bjdata(out_single) == j_single); + + // in-range boundary values still use the compact ndarray encoding + json const j_uint8_ok = json({{"_ArrayType_", "uint8"}, {"_ArraySize_", {2}}, {"_ArrayData_", {0, 255}}}); + CHECK(json::to_bjdata(j_uint8_ok) == std::vector({'[', '$', 'U', '#', '[', 'i', 2, ']', 0, 255})); + + json const j_int8_ok = json({{"_ArrayType_", "int8"}, {"_ArraySize_", {2}}, {"_ArrayData_", {-128, 127}}}); + CHECK(json::to_bjdata(j_int8_ok) == std::vector({'[', '$', 'i', '#', '[', 'i', 2, ']', 0x80, 0x7F})); + + json const j_single_ok = json({{"_ArrayType_", "single"}, {"_ArraySize_", {1}}, {"_ArrayData_", {1.5}}}); + const auto out_single_ok = json::to_bjdata(j_single_ok); + CHECK(out_single_ok.at(0) == '['); + CHECK(json::from_bjdata(out_single_ok) == json({1.5f})); + } } } From 35cdd5f4085c05618f532d24124bc2d4b7e903ac Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Fri, 11 Sep 2026 08:18:43 +0200 Subject: [PATCH 05/18] De-duplicate the diagnostic-positions test files via define-based recompilation (#5474) tests/src/unit-class_parser_diagnostic_positions.cpp and tests/src/unit-diagnostic-positions-only.cpp were maintained as near-copies of unit-class_parser.cpp and unit-diagnostic-positions.cpp respectively, and had drifted: trailing-comma handling, the #5342 filter-array/filter-value sections, and the cross-input-adapter diagnostics test were never ported to the positions-enabled copy. Fold the position-specific assertions into the base files, guarded by file a second time with the relevant macro set via CMake COMPILE_DEFINITIONS (mirroring the existing test-comparison_legacy pattern) instead of maintaining a separate source file. This removes the duplication and, as a side effect, closes the coverage gaps above since the full test file now compiles under JSON_DIAGNOSTIC_POSITIONS=1 as well. Fixes #5417 Signed-off-by: Niels Lohmann --- tests/CMakeLists.txt | 18 + tests/src/unit-class_parser.cpp | 266 +++ ...unit-class_parser_diagnostic_positions.cpp | 1960 ----------------- tests/src/unit-diagnostic-positions-only.cpp | 44 - tests/src/unit-diagnostic-positions.cpp | 14 +- 5 files changed, 297 insertions(+), 2005 deletions(-) delete mode 100644 tests/src/unit-class_parser_diagnostic_positions.cpp delete mode 100644 tests/src/unit-diagnostic-positions-only.cpp diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 322e40d80..86b4825d7 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -298,6 +298,24 @@ json_test_add_test_for(src/unit-comparison.cpp MAIN test_main CXX_STANDARDS ${test_cxx_standards} ${test_force} ) +# test the parser again with JSON_DIAGNOSTIC_POSITIONS enabled +json_test_set_test_options(test-class_parser_diagnostic_positions + COMPILE_DEFINITIONS JSON_DIAGNOSTIC_POSITIONS=1 +) +json_test_add_test_for(src/unit-class_parser.cpp + NAME test-class_parser_diagnostic_positions + MAIN test_main CXX_STANDARDS ${test_cxx_standards} ${test_force} +) + +# test diagnostic positions again without regular diagnostics (JSON pointer paths) +json_test_set_test_options(test-diagnostic-positions_only + COMPILE_DEFINITIONS JSON_DIAGNOSTICS=0 +) +json_test_add_test_for(src/unit-diagnostic-positions.cpp + NAME test-diagnostic-positions_only + MAIN test_main CXX_STANDARDS ${test_cxx_standards} ${test_force} +) + # *DO NOT* use json_test_set_test_options() below this line ############################################################################# diff --git a/tests/src/unit-class_parser.cpp b/tests/src/unit-class_parser.cpp index bb9cdcdc7..e0b21d975 100644 --- a/tests/src/unit-class_parser.cpp +++ b/tests/src/unit-class_parser.cpp @@ -346,6 +346,50 @@ void trailing_comma_helper(const std::string& s) } } +#if JSON_DIAGNOSTIC_POSITIONS +/** + * Validates that the generated JSON object is the same as expected + * Validates that the start position and end position match the start and end of the string + * + * This check assumes that there is no whitespace around the json object in the original string. + */ +void validate_generated_json_and_start_end_pos_helper(const std::string& original_string, const json& j, const json& check) +{ + CHECK(j == check); + CHECK(j.start_pos() == 0); + CHECK(j.end_pos() == original_string.size()); +} + +/** + * Parses the root object from the given root string and validates that the start and end positions for the nested object are correct. + * + * This checks that whitespace around the nested object is included in the start and end positions of the root object. + */ +void validate_start_end_pos_for_nested_obj_helper(const std::string& nested_type_json_str, const std::string& root_type_json_str, const json& expected_json, const json::parser_callback_t& cb = nullptr) +{ + json j; + + // 1. If callback is provided, use callback version of parse() + if (cb) + { + j = json::parse(root_type_json_str, cb); + } + else + { + j = json::parse(root_type_json_str); + } + + // 2. Check if the generated JSON is as expected + // Assumptions: The root_type_json_str does not have any whitespace around the json object + validate_generated_json_and_start_end_pos_helper(root_type_json_str, j, expected_json); + + // 3. Get the nested object + const auto& nested = j["nested"]; + // 4. Check if the start and end positions are generated correctly for nested objects and arrays + CHECK(nested_type_json_str == root_type_json_str.substr(nested.start_pos(), nested.end_pos() - nested.start_pos())); +} +#endif + } // namespace TEST_CASE("parser class") @@ -1939,6 +1983,228 @@ TEST_CASE("parser class") CHECK_THROWS_WITH_AS(_ = json::parse("/a", nullptr, true, true), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid comment; expecting '/' or '*' after '/'; last read: '/a'", json::parse_error); CHECK_THROWS_WITH_AS(_ = json::parse("/*", nullptr, true, true), "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - invalid comment; missing closing '*/'; last read: '/*'", json::parse_error); } + +#if JSON_DIAGNOSTIC_POSITIONS + // Macro for all test cases for start_pos and end_pos +#define SETUP_TESTCASES() \ + SECTION("with callback") \ + { \ + SECTION("filter nothing") \ + { \ + json::parser_callback_t const cb = [](int /*unused*/, json::parse_event_t /*unused*/, json& /*unused*/) noexcept \ + { \ + return true; \ + }; \ + validate_start_end_pos_for_nested_obj_helper(nested_type_json_str, root_type_json_str, expected, cb); \ + } \ + SECTION("filter element") \ + { \ + json::parser_callback_t const cb = [](int /*unused*/, json::parse_event_t event, json& j) noexcept \ + { \ + return (event != json::parse_event_t::key && event != json::parse_event_t::value) || j != json("a"); \ + }; \ + validate_start_end_pos_for_nested_obj_helper(nested_type_json_str, root_type_json_str, filteredExpected, cb); \ + } \ + } \ + SECTION("without callback") \ + { \ + validate_start_end_pos_for_nested_obj_helper(nested_type_json_str, root_type_json_str, expected); \ + } + + SECTION("retrieve start position and end position") + { + SECTION("for object") + { + // Create an object with spaces to test the start and end positions. Spaces will not be included in the + // JSON object, however, the start and end positions should include the spaces from the input JSON string. + const std::string nested_type_json_str = R"({ "a": 1,"b" : "test1"})"; + const std::string root_type_json_str = R"({ "nested": )" + nested_type_json_str + R"(, "anotherValue": "test2"})"; + auto expected = json({{"nested", {{"a", 1}, {"b", "test1"}}}, {"anotherValue", "test2"}}); + auto filteredExpected = expected; + filteredExpected["nested"].erase("a"); + + SETUP_TESTCASES() + } + + SECTION("for array") + { + const std::string nested_type_json_str = R"(["a", "test", 45])"; + const std::string root_type_json_str = R"({ "nested": )" + nested_type_json_str + R"(, "anotherValue": "test" })"; + auto expected = json({{"nested", {"a", "test", 45}}, {"anotherValue", "test"}}); + auto filteredExpected = expected; + filteredExpected["nested"] = json({"test", 45}); + SETUP_TESTCASES() + } + + SECTION("for array with objects") + { + const std::string nested_type_json_str = R"([{"a": 1, "b": "test"}, {"c": 2, "d": "test2"}])"; + const std::string root_type_json_str = R"({ "nested": )" + nested_type_json_str + R"(, "anotherValue": "test" })"; + auto expected = json({{"nested", {{{"a", 1}, {"b", "test"}}, {{"c", 2}, {"d", "test2"}}}}, {"anotherValue", "test"}}); + auto filteredExpected = expected; + filteredExpected["nested"][0].erase("a"); + SETUP_TESTCASES() + + auto j = json::parse(root_type_json_str); + auto nested_array = j["nested"]; + const auto& nested_obj = nested_array[0]; + CHECK(nested_type_json_str.substr(1, 21) == root_type_json_str.substr(nested_obj.start_pos(), nested_obj.end_pos() - nested_obj.start_pos())); + CHECK(nested_type_json_str.substr(24, 22) == root_type_json_str.substr(nested_array[1].start_pos(), nested_array[1].end_pos() - nested_array[1].start_pos())); + } + + SECTION("for two levels of nesting objects") + { + const std::string nested_type_json_str = R"({"nested2": {"b": "test"}})"; + const std::string root_type_json_str = R"({ "a": 2, "nested": )" + nested_type_json_str + R"(, "anotherValue": "test" })"; + auto expected = json({{"a", 2}, {"nested", {{"nested2", {{"b", "test"}}}}}, {"anotherValue", "test"}}); + auto filteredExpected = expected; + filteredExpected.erase("a"); + SETUP_TESTCASES() + + auto j = json::parse(root_type_json_str); + auto nested_obj = j["nested"]["nested2"]; + CHECK(nested_type_json_str.substr(12, 13) == root_type_json_str.substr(nested_obj.start_pos(), nested_obj.end_pos() - nested_obj.start_pos())); + } + + SECTION("for simple types") + { + SECTION("no nested") + { + SECTION("with callback") + { + json::parser_callback_t const cb = [](int /*unused*/, json::parse_event_t /*unused*/, json& /*unused*/) noexcept + { + return true; + }; + + // 1. string type + std::string json_str = R"("test")"; + auto j = json::parse(json_str, cb); + validate_generated_json_and_start_end_pos_helper(json_str, j, "test"); + + // 2. number type + json_str = R"(1)"; + j = json::parse(json_str, cb); + validate_generated_json_and_start_end_pos_helper(json_str, j, 1); + + // 3. boolean type + json_str = R"(true)"; + j = json::parse(json_str, cb); + validate_generated_json_and_start_end_pos_helper(json_str, j, true); + + // 4. null type + json_str = R"(null)"; + j = json::parse(json_str, cb); + validate_generated_json_and_start_end_pos_helper(json_str, j, nullptr); + } + + SECTION("without callback") + { + // 1. string type + std::string json_str = R"("test")"; + auto j = json::parse(json_str); + validate_generated_json_and_start_end_pos_helper(json_str, j, "test"); + + // 2. number type + json_str = R"(1)"; + j = json::parse(json_str); + validate_generated_json_and_start_end_pos_helper(json_str, j, 1); + + json_str = R"(1.001239923)"; + j = json::parse(json_str); + validate_generated_json_and_start_end_pos_helper(json_str, j, 1.001239923); + + json_str = R"(1.123812389000000)"; + j = json::parse(json_str); + validate_generated_json_and_start_end_pos_helper(json_str, j, 1.123812389); + + // 3. boolean type + json_str = R"(true)"; + j = json::parse(json_str); + validate_generated_json_and_start_end_pos_helper(json_str, j, true); + + json_str = R"(false)"; + j = json::parse(json_str); + validate_generated_json_and_start_end_pos_helper(json_str, j, false); + + // 4. null type + json_str = R"(null)"; + j = json::parse(json_str); + validate_generated_json_and_start_end_pos_helper(json_str, j, nullptr); + } + } + + SECTION("string type") + { + const std::string nested_type_json_str = R"("test")"; + const std::string root_type_json_str = R"({ "a": 1, "nested": )" + nested_type_json_str + R"(, "anotherValue": "test" })"; + auto expected = json({{"nested", "test"}, {"anotherValue", "test"}, {"a", 1}}); + auto filteredExpected = expected; + filteredExpected.erase("a"); + SETUP_TESTCASES() + } + + SECTION("number type") + { + const std::string nested_type_json_str = R"(2)"; + const std::string root_type_json_str = R"({ "a": 1, "nested": )" + nested_type_json_str + R"(, "anotherValue": "test" })"; + auto expected = json({{"nested", 2}, {"anotherValue", "test"}, {"a", 1}}); + auto filteredExpected = expected; + filteredExpected.erase("a"); + SETUP_TESTCASES() + } + + SECTION("boolean type") + { + const std::string nested_type_json_str = R"(true)"; + const std::string root_type_json_str = R"({ "a": 1, "nested": )" + nested_type_json_str + R"(, "anotherValue": "test" })"; + auto expected = json({{"nested", true}, {"anotherValue", "test"}, {"a", 1}}); + auto filteredExpected = expected; + filteredExpected.erase("a"); + SETUP_TESTCASES() + } + + SECTION("null type") + { + const std::string nested_type_json_str = R"(null)"; + const std::string root_type_json_str = R"({ "a": 1, "nested": )" + nested_type_json_str + R"(, "anotherValue": "test" })"; + auto expected = json({{"nested", nullptr}, {"anotherValue", "test"}, {"a", 1}}); + auto filteredExpected = expected; + filteredExpected.erase("a"); + SETUP_TESTCASES() + } + } + SECTION("with leading whitespace and newlines around root JSON") + { + const std::string initial_whitespace = R"( + + )"; + const std::string nested_type_json_str = R"({ + "a": 1, + "nested": { + "b": "test" + }, + "anotherValue": "test" + })"; + const std::string end_whitespace = R"( + + )"; + const std::string root_type_json_str = initial_whitespace + nested_type_json_str + end_whitespace; + + auto expected = json({{"a", 1}, {"nested", {{"b", "test"}}}, {"anotherValue", "test"}}); + + auto j = json::parse(root_type_json_str); + + // 2. Check if the generated JSON is as expected + CHECK(j == expected); + + // 3. Check if the start and end positions do not include the surrounding whitespace + CHECK(j.start_pos() == initial_whitespace.size()); + CHECK(j.end_pos() == root_type_json_str.size() - end_whitespace.size()); + } + } +#undef SETUP_TESTCASES +#endif } // this test relies on parse errors being thrown, so it is skipped when diff --git a/tests/src/unit-class_parser_diagnostic_positions.cpp b/tests/src/unit-class_parser_diagnostic_positions.cpp deleted file mode 100644 index 794182f30..000000000 --- a/tests/src/unit-class_parser_diagnostic_positions.cpp +++ /dev/null @@ -1,1960 +0,0 @@ -// __ _____ _____ _____ -// __| | __| | | | JSON for Modern C++ (supporting code) -// | | |__ | | | | | | version 3.12.0 -// |_____|_____|_____|_|___| https://github.com/nlohmann/json -// -// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann -// SPDX-License-Identifier: MIT - -#include "doctest_compatibility.h" -#define JSON_TESTS_PRIVATE -#ifdef JSON_DIAGNOSTIC_POSITIONS - #undef JSON_DIAGNOSTIC_POSITIONS -#endif - -#define JSON_DIAGNOSTIC_POSITIONS 1 -#include -using nlohmann::json; - -#ifdef JSON_TEST_NO_GLOBAL_UDLS - using namespace nlohmann::literals; // NOLINT(google-build-using-namespace) -#endif - -#include - -#include "test_utils.hpp" - -namespace -{ -class SaxEventLogger -{ - public: - bool null() - { - events.emplace_back("null()"); - return true; - } - - bool boolean(bool val) - { - events.emplace_back(val ? "boolean(true)" : "boolean(false)"); - return true; - } - - bool number_integer(json::number_integer_t val) - { - events.push_back("number_integer(" + std::to_string(val) + ")"); - return true; - } - - bool number_unsigned(json::number_unsigned_t val) - { - events.push_back("number_unsigned(" + std::to_string(val) + ")"); - return true; - } - - bool number_float(json::number_float_t /*unused*/, const std::string& s) - { - events.push_back("number_float(" + s + ")"); - return true; - } - - bool string(std::string& val) - { - events.push_back("string(" + val + ")"); - return true; - } - - bool binary(json::binary_t& val) - { - std::string binary_contents = "binary("; - std::string comma_space; - for (auto b : val) - { - binary_contents.append(comma_space); - binary_contents.append(std::to_string(static_cast(b))); - comma_space = ", "; - } - binary_contents.append(")"); - events.push_back(binary_contents); - return true; - } - - bool start_object(std::size_t elements) - { - if (elements == (std::numeric_limits::max)()) - { - events.emplace_back("start_object()"); - } - else - { - events.push_back("start_object(" + std::to_string(elements) + ")"); - } - return true; - } - - bool key(std::string& val) - { - events.push_back("key(" + val + ")"); - return true; - } - - bool end_object() - { - events.emplace_back("end_object()"); - return true; - } - - bool start_array(std::size_t elements) - { - if (elements == (std::numeric_limits::max)()) - { - events.emplace_back("start_array()"); - } - else - { - events.push_back("start_array(" + std::to_string(elements) + ")"); - } - return true; - } - - bool end_array() - { - events.emplace_back("end_array()"); - return true; - } - - bool parse_error(std::size_t position, const std::string& /*unused*/, const json::exception& /*unused*/) - { - errored = true; - events.push_back("parse_error(" + std::to_string(position) + ")"); - return false; - } - - std::vector events {}; // NOLINT(readability-redundant-member-init) - bool errored = false; -}; - -class SaxCountdown : public nlohmann::json::json_sax_t -{ - public: - explicit SaxCountdown(const int count) : events_left(count) - {} - - bool null() override - { - return events_left-- > 0; - } - - bool boolean(bool /*val*/) override - { - return events_left-- > 0; - } - - bool number_integer(json::number_integer_t /*val*/) override - { - return events_left-- > 0; - } - - bool number_unsigned(json::number_unsigned_t /*val*/) override - { - return events_left-- > 0; - } - - bool number_float(json::number_float_t /*val*/, const std::string& /*s*/) override - { - return events_left-- > 0; - } - - bool string(std::string& /*val*/) override - { - return events_left-- > 0; - } - - bool binary(json::binary_t& /*val*/) override - { - return events_left-- > 0; - } - - bool start_object(std::size_t /*elements*/) override - { - return events_left-- > 0; - } - - bool key(std::string& /*val*/) override - { - return events_left-- > 0; - } - - bool end_object() override - { - return events_left-- > 0; - } - - bool start_array(std::size_t /*elements*/) override - { - return events_left-- > 0; - } - - bool end_array() override - { - return events_left-- > 0; - } - - bool parse_error(std::size_t /*position*/, const std::string& /*last_token*/, const json::exception& /*ex*/) override - { - return false; - } - - private: - int events_left = 0; -}; - -json parser_helper(const std::string& s); -bool accept_helper(const std::string& s); -void comments_helper(const std::string& s); - -json parser_helper(const std::string& s) -{ - json j; - json::parser(nlohmann::detail::input_adapter(s)).parse(true, j); - - // if this line was reached, no exception occurred - // -> check if result is the same without exceptions - json j_nothrow; - CHECK_NOTHROW(json::parser(nlohmann::detail::input_adapter(s), nullptr, false).parse(true, j_nothrow)); - CHECK(j_nothrow == j); - - json j_sax; - nlohmann::detail::json_sax_dom_parser sdp(j_sax); - json::sax_parse(s, &sdp); - CHECK(j_sax == j); - - comments_helper(s); - - return j; -} - -bool accept_helper(const std::string& s) -{ - CAPTURE(s) - - // 1. parse s without exceptions - json j; - CHECK_NOTHROW(json::parser(nlohmann::detail::input_adapter(s), nullptr, false).parse(true, j)); - const bool ok_noexcept = !j.is_discarded(); - - // 2. accept s - const bool ok_accept = json::parser(nlohmann::detail::input_adapter(s)).accept(true); - - // 3. check if both approaches come to the same result - CHECK(ok_noexcept == ok_accept); - - // 4. parse with SAX (compare with relaxed accept result) - SaxEventLogger el; - CHECK_NOTHROW(json::sax_parse(s, &el, json::input_format_t::json, false)); - CHECK(json::parser(nlohmann::detail::input_adapter(s)).accept(false) == !el.errored); - - // 5. parse with simple callback - json::parser_callback_t const cb = [](int /*unused*/, json::parse_event_t /*unused*/, json& /*unused*/) noexcept - { - return true; - }; - json const j_cb = json::parse(s, cb, false); - const bool ok_noexcept_cb = !j_cb.is_discarded(); - - // 6. check if this approach came to the same result - CHECK(ok_noexcept == ok_noexcept_cb); - - // 7. check if comments are properly ignored - if (ok_accept) - { - comments_helper(s); - } - - // 8. return result - return ok_accept; -} - -void comments_helper(const std::string& s) -{ - json _; - - // parse/accept with default parser - CHECK_NOTHROW(_ = json::parse(s)); - CHECK(json::accept(s)); - - // parse/accept while skipping comments - CHECK_NOTHROW(_ = json::parse(s, nullptr, false, true)); - CHECK(json::accept(s, true)); - - std::vector json_with_comments; - - // start with a comment - json_with_comments.push_back(std::string("// this is a comment\n") + s); - json_with_comments.push_back(std::string("/* this is a comment */") + s); - // end with a comment - json_with_comments.push_back(s + "// this is a comment"); - json_with_comments.push_back(s + "/* this is a comment */"); - - // check all strings - for (const auto& json_with_comment : json_with_comments) - { - CAPTURE(json_with_comment) - CHECK_THROWS_AS(_ = json::parse(json_with_comment), json::parse_error); - CHECK(!json::accept(json_with_comment)); - - CHECK_NOTHROW(_ = json::parse(json_with_comment, nullptr, true, true)); - CHECK(json::accept(json_with_comment, true)); - } -} - -/** - * Validates that the generated JSON object is the same as expected - * Validates that the start position and end position match the start and end of the string - * - * This check assumes that there is no whitespace around the json object in the original string. - */ -void validate_generated_json_and_start_end_pos_helper(const std::string& original_string, const json& j, const json& check) -{ - CHECK(j == check); - CHECK(j.start_pos() == 0); - CHECK(j.end_pos() == original_string.size()); -} - -/** - * Parses the root object from the given root string and validates that the start and end positions for the nested object are correct. - * - * This checks that whitespace around the nested object is included in the start and end positions of the root object. - */ -void validate_start_end_pos_for_nested_obj_helper(const std::string& nested_type_json_str, const std::string& root_type_json_str, const json& expected_json, const json::parser_callback_t& cb = nullptr) -{ - json j; - - // 1. If callback is provided, use callback version of parse() - if (cb) - { - j = json::parse(root_type_json_str, cb); - } - else - { - j = json::parse(root_type_json_str); - } - - // 2. Check if the generated JSON is as expected - // Assumptions: The root_type_json_str does not have any whitespace around the json object - validate_generated_json_and_start_end_pos_helper(root_type_json_str, j, expected_json); - - // 3. Get the nested object - const auto& nested = j["nested"]; - // 4. Check if the start and end positions are generated correctly for nested objects and arrays - CHECK(nested_type_json_str == root_type_json_str.substr(nested.start_pos(), nested.end_pos() - nested.start_pos())); -} - -} // namespace - -TEST_CASE("parser class") -{ - SECTION("parse") - { - SECTION("null") - { - CHECK(parser_helper("null") == json(nullptr)); - } - - SECTION("true") - { - CHECK(parser_helper("true") == json(true)); - } - - SECTION("false") - { - CHECK(parser_helper("false") == json(false)); - } - - SECTION("array") - { - SECTION("empty array") - { - CHECK(parser_helper("[]") == json(json::value_t::array)); - CHECK(parser_helper("[ ]") == json(json::value_t::array)); - } - - SECTION("nonempty array") - { - CHECK(parser_helper("[true, false, null]") == json({true, false, nullptr})); - } - } - - SECTION("object") - { - SECTION("empty object") - { - CHECK(parser_helper("{}") == json(json::value_t::object)); - CHECK(parser_helper("{ }") == json(json::value_t::object)); - } - - SECTION("nonempty object") - { - CHECK(parser_helper("{\"\": true, \"one\": 1, \"two\": null}") == json({{"", true}, {"one", 1}, {"two", nullptr}})); - } - } - - SECTION("string") - { - // empty string - CHECK(parser_helper("\"\"") == json(json::value_t::string)); - - SECTION("errors") - { - // error: tab in string - CHECK_THROWS_WITH_AS(parser_helper("\"\t\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+0009 (HT) must be escaped to \\u0009 or \\t; last read: '\"'", json::parse_error&); - // error: newline in string - CHECK_THROWS_WITH_AS(parser_helper("\"\n\""), "[json.exception.parse_error.101] parse error at line 2, column 0: syntax error while parsing value - invalid string: control character U+000A (LF) must be escaped to \\u000A or \\n; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\r\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+000D (CR) must be escaped to \\u000D or \\r; last read: '\"'", json::parse_error&); - // error: backspace in string - CHECK_THROWS_WITH_AS(parser_helper("\"\b\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+0008 (BS) must be escaped to \\u0008 or \\b; last read: '\"'", json::parse_error&); - // improve code coverage - CHECK_THROWS_AS(parser_helper("\uFF01"), json::parse_error&); - CHECK_THROWS_AS(parser_helper("[-4:1,]"), json::parse_error&); - // unescaped control characters - CHECK_THROWS_WITH_AS(parser_helper("\"\x00\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: missing closing quote; last read: '\"'", json::parse_error&); // NOLINT(bugprone-string-literal-with-embedded-nul) - CHECK_THROWS_WITH_AS(parser_helper("\"\x01\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+0001 (SOH) must be escaped to \\u0001; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x02\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+0002 (STX) must be escaped to \\u0002; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x03\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+0003 (ETX) must be escaped to \\u0003; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x04\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+0004 (EOT) must be escaped to \\u0004; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x05\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+0005 (ENQ) must be escaped to \\u0005; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x06\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+0006 (ACK) must be escaped to \\u0006; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x07\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+0007 (BEL) must be escaped to \\u0007; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x08\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+0008 (BS) must be escaped to \\u0008 or \\b; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x09\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+0009 (HT) must be escaped to \\u0009 or \\t; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x0a\""), "[json.exception.parse_error.101] parse error at line 2, column 0: syntax error while parsing value - invalid string: control character U+000A (LF) must be escaped to \\u000A or \\n; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x0b\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+000B (VT) must be escaped to \\u000B; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x0c\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+000C (FF) must be escaped to \\u000C or \\f; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x0d\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+000D (CR) must be escaped to \\u000D or \\r; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x0e\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+000E (SO) must be escaped to \\u000E; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x0f\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+000F (SI) must be escaped to \\u000F; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x10\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+0010 (DLE) must be escaped to \\u0010; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x11\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+0011 (DC1) must be escaped to \\u0011; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x12\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+0012 (DC2) must be escaped to \\u0012; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x13\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+0013 (DC3) must be escaped to \\u0013; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x14\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+0014 (DC4) must be escaped to \\u0014; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x15\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+0015 (NAK) must be escaped to \\u0015; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x16\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+0016 (SYN) must be escaped to \\u0016; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x17\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+0017 (ETB) must be escaped to \\u0017; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x18\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+0018 (CAN) must be escaped to \\u0018; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x19\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+0019 (EM) must be escaped to \\u0019; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x1a\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+001A (SUB) must be escaped to \\u001A; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x1b\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+001B (ESC) must be escaped to \\u001B; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x1c\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+001C (FS) must be escaped to \\u001C; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x1d\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+001D (GS) must be escaped to \\u001D; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x1e\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+001E (RS) must be escaped to \\u001E; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\x1f\""), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+001F (US) must be escaped to \\u001F; last read: '\"'", json::parse_error&); - - SECTION("additional test for null byte") - { - // The test above for the null byte is wrong, because passing - // a string to the parser only reads int until it encounters - // a null byte. This test inserts the null byte later on and - // uses an iterator range. - std::string s = "\"1\""; - s[1] = '\0'; - json _; - CHECK_THROWS_WITH_AS(_ = json::parse(s.begin(), s.end()), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: control character U+0000 (NUL) must be escaped to \\u0000; last read: '\"'", json::parse_error&); - } - } - - SECTION("escaped") - { - // quotation mark "\"" - auto r1 = R"("\"")"_json; - CHECK(parser_helper("\"\\\"\"") == r1); - // reverse solidus "\\" - auto r2 = R"("\\")"_json; - CHECK(parser_helper("\"\\\\\"") == r2); - // solidus - CHECK(parser_helper("\"\\/\"") == R"("/")"_json); - // backspace - CHECK(parser_helper("\"\\b\"") == json("\b")); - // formfeed - CHECK(parser_helper("\"\\f\"") == json("\f")); - // newline - CHECK(parser_helper("\"\\n\"") == json("\n")); - // carriage return - CHECK(parser_helper("\"\\r\"") == json("\r")); - // horizontal tab - CHECK(parser_helper("\"\\t\"") == json("\t")); - - CHECK(parser_helper("\"\\u0001\"").get() == "\x01"); - CHECK(parser_helper("\"\\u000a\"").get() == "\n"); - CHECK(parser_helper("\"\\u00b0\"").get() == "°"); - CHECK(parser_helper("\"\\u0c00\"").get() == "ఀ"); - CHECK(parser_helper("\"\\ud000\"").get() == "퀀"); - CHECK(parser_helper("\"\\u000E\"").get() == "\x0E"); - CHECK(parser_helper("\"\\u00F0\"").get() == "ð"); - CHECK(parser_helper("\"\\u0100\"").get() == "Ā"); - CHECK(parser_helper("\"\\u2000\"").get() == " "); - CHECK(parser_helper("\"\\uFFFF\"").get() == "￿"); - CHECK(parser_helper("\"\\u20AC\"").get() == "€"); - CHECK(parser_helper("\"€\"").get() == "€"); - CHECK(parser_helper("\"🎈\"").get() == "🎈"); - - CHECK(parser_helper("\"\\ud80c\\udc60\"").get() == "\xf0\x93\x81\xa0"); - CHECK(parser_helper("\"\\ud83c\\udf1e\"").get() == "🌞"); - } - } - - SECTION("number") - { - SECTION("integers") - { - SECTION("without exponent") - { - CHECK(parser_helper("-128") == json(-128)); - CHECK(parser_helper("-0") == json(-0)); - CHECK(parser_helper("0") == json(0)); - CHECK(parser_helper("128") == json(128)); - } - - SECTION("with exponent") - { - CHECK(parser_helper("0e1") == json(0e1)); - CHECK(parser_helper("0E1") == json(0e1)); - - CHECK(parser_helper("10000E-4") == json(10000e-4)); - CHECK(parser_helper("10000E-3") == json(10000e-3)); - CHECK(parser_helper("10000E-2") == json(10000e-2)); - CHECK(parser_helper("10000E-1") == json(10000e-1)); - CHECK(parser_helper("10000E0") == json(10000e0)); - CHECK(parser_helper("10000E1") == json(10000e1)); - CHECK(parser_helper("10000E2") == json(10000e2)); - CHECK(parser_helper("10000E3") == json(10000e3)); - CHECK(parser_helper("10000E4") == json(10000e4)); - - CHECK(parser_helper("10000e-4") == json(10000e-4)); - CHECK(parser_helper("10000e-3") == json(10000e-3)); - CHECK(parser_helper("10000e-2") == json(10000e-2)); - CHECK(parser_helper("10000e-1") == json(10000e-1)); - CHECK(parser_helper("10000e0") == json(10000e0)); - CHECK(parser_helper("10000e1") == json(10000e1)); - CHECK(parser_helper("10000e2") == json(10000e2)); - CHECK(parser_helper("10000e3") == json(10000e3)); - CHECK(parser_helper("10000e4") == json(10000e4)); - - CHECK(parser_helper("-0e1") == json(-0e1)); - CHECK(parser_helper("-0E1") == json(-0e1)); - CHECK(parser_helper("-0E123") == json(-0e123)); - - // numbers after exponent - CHECK(parser_helper("10E0") == json(10e0)); - CHECK(parser_helper("10E1") == json(10e1)); - CHECK(parser_helper("10E2") == json(10e2)); - CHECK(parser_helper("10E3") == json(10e3)); - CHECK(parser_helper("10E4") == json(10e4)); - CHECK(parser_helper("10E5") == json(10e5)); - CHECK(parser_helper("10E6") == json(10e6)); - CHECK(parser_helper("10E7") == json(10e7)); - CHECK(parser_helper("10E8") == json(10e8)); - CHECK(parser_helper("10E9") == json(10e9)); - CHECK(parser_helper("10E+0") == json(10e0)); - CHECK(parser_helper("10E+1") == json(10e1)); - CHECK(parser_helper("10E+2") == json(10e2)); - CHECK(parser_helper("10E+3") == json(10e3)); - CHECK(parser_helper("10E+4") == json(10e4)); - CHECK(parser_helper("10E+5") == json(10e5)); - CHECK(parser_helper("10E+6") == json(10e6)); - CHECK(parser_helper("10E+7") == json(10e7)); - CHECK(parser_helper("10E+8") == json(10e8)); - CHECK(parser_helper("10E+9") == json(10e9)); - CHECK(parser_helper("10E-1") == json(10e-1)); - CHECK(parser_helper("10E-2") == json(10e-2)); - CHECK(parser_helper("10E-3") == json(10e-3)); - CHECK(parser_helper("10E-4") == json(10e-4)); - CHECK(parser_helper("10E-5") == json(10e-5)); - CHECK(parser_helper("10E-6") == json(10e-6)); - CHECK(parser_helper("10E-7") == json(10e-7)); - CHECK(parser_helper("10E-8") == json(10e-8)); - CHECK(parser_helper("10E-9") == json(10e-9)); - } - - SECTION("edge cases") - { - // From RFC8259, Section 6: - // Note that when such software is used, numbers that are - // integers and are in the range [-(2**53)+1, (2**53)-1] - // are interoperable in the sense that implementations will - // agree exactly on their numeric values. - - // -(2**53)+1 - CHECK(parser_helper("-9007199254740991").get() == -9007199254740991); - // (2**53)-1 - CHECK(parser_helper("9007199254740991").get() == 9007199254740991); - } - - SECTION("over the edge cases") // issue #178 - Integer conversion to unsigned (incorrect handling of 64-bit integers) - { - // While RFC8259, Section 6 specifies a preference for support - // for ranges in range of IEEE 754-2008 binary64 (double precision) - // this does not accommodate 64-bit integers without loss of accuracy. - // As 64-bit integers are now widely used in software, it is desirable - // to expand support to the full 64 bit (signed and unsigned) range - // i.e. -(2**63) -> (2**64)-1. - - // -(2**63) ** Note: compilers see negative literals as negated positive numbers (hence the -1)) - CHECK(parser_helper("-9223372036854775808").get() == -9223372036854775807 - 1); - // (2**63)-1 - CHECK(parser_helper("9223372036854775807").get() == 9223372036854775807); - // (2**64)-1 - CHECK(parser_helper("18446744073709551615").get() == 18446744073709551615u); - } - } - - SECTION("floating-point") - { - SECTION("without exponent") - { - CHECK(parser_helper("-128.5") == json(-128.5)); - CHECK(parser_helper("0.999") == json(0.999)); - CHECK(parser_helper("128.5") == json(128.5)); - CHECK(parser_helper("-0.0") == json(-0.0)); - } - - SECTION("with exponent") - { - CHECK(parser_helper("-128.5E3") == json(-128.5E3)); - CHECK(parser_helper("-128.5E-3") == json(-128.5E-3)); - CHECK(parser_helper("-0.0e1") == json(-0.0e1)); - CHECK(parser_helper("-0.0E1") == json(-0.0e1)); - } - } - - SECTION("overflow") - { - // overflows during parsing yield an exception - // empty() is nodiscard; the exception is thrown by parser_helper() itself, before empty() would run - CHECK_THROWS_WITH_AS(utils::ignore_return_value(parser_helper("1.18973e+4932").empty()), "[json.exception.out_of_range.406] number overflow parsing '1.18973e+4932'", json::out_of_range&); - } - - SECTION("invalid numbers") - { - // numbers must not begin with "+" - CHECK_THROWS_AS(parser_helper("+1"), json::parse_error&); - CHECK_THROWS_AS(parser_helper("+0"), json::parse_error&); - - CHECK_THROWS_WITH_AS(parser_helper("01"), - "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - unexpected number literal; expected end of input", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("-01"), - "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - unexpected number literal; expected end of input", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("--1"), - "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid number; expected digit after '-'; last read: '--'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("1."), - "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - invalid number; expected digit after '.'; last read: '1.'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("1E"), - "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - invalid number; expected '+', '-', or digit after exponent; last read: '1E'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("1E-"), - "[json.exception.parse_error.101] parse error at line 1, column 4: syntax error while parsing value - invalid number; expected digit after exponent sign; last read: '1E-'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("1.E1"), - "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - invalid number; expected digit after '.'; last read: '1.E'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("-1E"), - "[json.exception.parse_error.101] parse error at line 1, column 4: syntax error while parsing value - invalid number; expected '+', '-', or digit after exponent; last read: '-1E'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("-0E#"), - "[json.exception.parse_error.101] parse error at line 1, column 4: syntax error while parsing value - invalid number; expected '+', '-', or digit after exponent; last read: '-0E#'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("-0E-#"), - "[json.exception.parse_error.101] parse error at line 1, column 5: syntax error while parsing value - invalid number; expected digit after exponent sign; last read: '-0E-#'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("-0#"), - "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - invalid literal; last read: '-0#'; expected end of input", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("-0.0:"), - "[json.exception.parse_error.101] parse error at line 1, column 5: syntax error while parsing value - unexpected ':'; expected end of input", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("-0.0Z"), - "[json.exception.parse_error.101] parse error at line 1, column 5: syntax error while parsing value - invalid literal; last read: '-0.0Z'; expected end of input", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("-0E123:"), - "[json.exception.parse_error.101] parse error at line 1, column 7: syntax error while parsing value - unexpected ':'; expected end of input", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("-0e0-:"), - "[json.exception.parse_error.101] parse error at line 1, column 6: syntax error while parsing value - invalid number; expected digit after '-'; last read: '-:'; expected end of input", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("-0e-:"), - "[json.exception.parse_error.101] parse error at line 1, column 5: syntax error while parsing value - invalid number; expected digit after exponent sign; last read: '-0e-:'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("-0f"), - "[json.exception.parse_error.101] parse error at line 1, column 4: syntax error while parsing value - invalid literal; last read: '-0f'; expected end of input", json::parse_error&); - } - } - } - - SECTION("accept") - { - SECTION("null") - { - CHECK(accept_helper("null")); - } - - SECTION("true") - { - CHECK(accept_helper("true")); - } - - SECTION("false") - { - CHECK(accept_helper("false")); - } - - SECTION("array") - { - SECTION("empty array") - { - CHECK(accept_helper("[]")); - CHECK(accept_helper("[ ]")); - } - - SECTION("nonempty array") - { - CHECK(accept_helper("[true, false, null]")); - } - } - - SECTION("object") - { - SECTION("empty object") - { - CHECK(accept_helper("{}")); - CHECK(accept_helper("{ }")); - } - - SECTION("nonempty object") - { - CHECK(accept_helper("{\"\": true, \"one\": 1, \"two\": null}")); - } - } - - SECTION("string") - { - // empty string - CHECK(accept_helper("\"\"")); - - SECTION("errors") - { - // error: tab in string - CHECK(accept_helper("\"\t\"") == false); - // error: newline in string - CHECK(accept_helper("\"\n\"") == false); - CHECK(accept_helper("\"\r\"") == false); - // error: backspace in string - CHECK(accept_helper("\"\b\"") == false); - // improve code coverage - CHECK(accept_helper("\uFF01") == false); - CHECK(accept_helper("[-4:1,]") == false); - // unescaped control characters - CHECK(accept_helper("\"\x00\"") == false); // NOLINT(bugprone-string-literal-with-embedded-nul) - CHECK(accept_helper("\"\x01\"") == false); - CHECK(accept_helper("\"\x02\"") == false); - CHECK(accept_helper("\"\x03\"") == false); - CHECK(accept_helper("\"\x04\"") == false); - CHECK(accept_helper("\"\x05\"") == false); - CHECK(accept_helper("\"\x06\"") == false); - CHECK(accept_helper("\"\x07\"") == false); - CHECK(accept_helper("\"\x08\"") == false); - CHECK(accept_helper("\"\x09\"") == false); - CHECK(accept_helper("\"\x0a\"") == false); - CHECK(accept_helper("\"\x0b\"") == false); - CHECK(accept_helper("\"\x0c\"") == false); - CHECK(accept_helper("\"\x0d\"") == false); - CHECK(accept_helper("\"\x0e\"") == false); - CHECK(accept_helper("\"\x0f\"") == false); - CHECK(accept_helper("\"\x10\"") == false); - CHECK(accept_helper("\"\x11\"") == false); - CHECK(accept_helper("\"\x12\"") == false); - CHECK(accept_helper("\"\x13\"") == false); - CHECK(accept_helper("\"\x14\"") == false); - CHECK(accept_helper("\"\x15\"") == false); - CHECK(accept_helper("\"\x16\"") == false); - CHECK(accept_helper("\"\x17\"") == false); - CHECK(accept_helper("\"\x18\"") == false); - CHECK(accept_helper("\"\x19\"") == false); - CHECK(accept_helper("\"\x1a\"") == false); - CHECK(accept_helper("\"\x1b\"") == false); - CHECK(accept_helper("\"\x1c\"") == false); - CHECK(accept_helper("\"\x1d\"") == false); - CHECK(accept_helper("\"\x1e\"") == false); - CHECK(accept_helper("\"\x1f\"") == false); - } - - SECTION("escaped") - { - // quotation mark "\"" - auto r1 = R"("\"")"_json; - CHECK(accept_helper("\"\\\"\"")); - // reverse solidus "\\" - auto r2 = R"("\\")"_json; - CHECK(accept_helper("\"\\\\\"")); - // solidus - CHECK(accept_helper("\"\\/\"")); - // backspace - CHECK(accept_helper("\"\\b\"")); - // formfeed - CHECK(accept_helper("\"\\f\"")); - // newline - CHECK(accept_helper("\"\\n\"")); - // carriage return - CHECK(accept_helper("\"\\r\"")); - // horizontal tab - CHECK(accept_helper("\"\\t\"")); - - CHECK(accept_helper("\"\\u0001\"")); - CHECK(accept_helper("\"\\u000a\"")); - CHECK(accept_helper("\"\\u00b0\"")); - CHECK(accept_helper("\"\\u0c00\"")); - CHECK(accept_helper("\"\\ud000\"")); - CHECK(accept_helper("\"\\u000E\"")); - CHECK(accept_helper("\"\\u00F0\"")); - CHECK(accept_helper("\"\\u0100\"")); - CHECK(accept_helper("\"\\u2000\"")); - CHECK(accept_helper("\"\\uFFFF\"")); - CHECK(accept_helper("\"\\u20AC\"")); - CHECK(accept_helper("\"€\"")); - CHECK(accept_helper("\"🎈\"")); - - CHECK(accept_helper("\"\\ud80c\\udc60\"")); - CHECK(accept_helper("\"\\ud83c\\udf1e\"")); - } - } - - SECTION("number") - { - SECTION("integers") - { - SECTION("without exponent") - { - CHECK(accept_helper("-128")); - CHECK(accept_helper("-0")); - CHECK(accept_helper("0")); - CHECK(accept_helper("128")); - } - - SECTION("with exponent") - { - CHECK(accept_helper("0e1")); - CHECK(accept_helper("0E1")); - - CHECK(accept_helper("10000E-4")); - CHECK(accept_helper("10000E-3")); - CHECK(accept_helper("10000E-2")); - CHECK(accept_helper("10000E-1")); - CHECK(accept_helper("10000E0")); - CHECK(accept_helper("10000E1")); - CHECK(accept_helper("10000E2")); - CHECK(accept_helper("10000E3")); - CHECK(accept_helper("10000E4")); - - CHECK(accept_helper("10000e-4")); - CHECK(accept_helper("10000e-3")); - CHECK(accept_helper("10000e-2")); - CHECK(accept_helper("10000e-1")); - CHECK(accept_helper("10000e0")); - CHECK(accept_helper("10000e1")); - CHECK(accept_helper("10000e2")); - CHECK(accept_helper("10000e3")); - CHECK(accept_helper("10000e4")); - - CHECK(accept_helper("-0e1")); - CHECK(accept_helper("-0E1")); - CHECK(accept_helper("-0E123")); - } - - SECTION("edge cases") - { - // From RFC8259, Section 6: - // Note that when such software is used, numbers that are - // integers and are in the range [-(2**53)+1, (2**53)-1] - // are interoperable in the sense that implementations will - // agree exactly on their numeric values. - - // -(2**53)+1 - CHECK(accept_helper("-9007199254740991")); - // (2**53)-1 - CHECK(accept_helper("9007199254740991")); - } - - SECTION("over the edge cases") // issue #178 - Integer conversion to unsigned (incorrect handling of 64-bit integers) - { - // While RFC8259, Section 6 specifies a preference for support - // for ranges in range of IEEE 754-2008 binary64 (double precision) - // this does not accommodate 64 bit integers without loss of accuracy. - // As 64 bit integers are now widely used in software, it is desirable - // to expand support to the full 64 bit (signed and unsigned) range - // i.e. -(2**63) -> (2**64)-1. - - // -(2**63) ** Note: compilers see negative literals as negated positive numbers (hence the -1)) - CHECK(accept_helper("-9223372036854775808")); - // (2**63)-1 - CHECK(accept_helper("9223372036854775807")); - // (2**64)-1 - CHECK(accept_helper("18446744073709551615")); - } - } - - SECTION("floating-point") - { - SECTION("without exponent") - { - CHECK(accept_helper("-128.5")); - CHECK(accept_helper("0.999")); - CHECK(accept_helper("128.5")); - CHECK(accept_helper("-0.0")); - } - - SECTION("with exponent") - { - CHECK(accept_helper("-128.5E3")); - CHECK(accept_helper("-128.5E-3")); - CHECK(accept_helper("-0.0e1")); - CHECK(accept_helper("-0.0E1")); - } - } - - SECTION("overflow") - { - // overflows during parsing - CHECK(!accept_helper("1.18973e+4932")); - } - - SECTION("invalid numbers") - { - CHECK(accept_helper("01") == false); - CHECK(accept_helper("--1") == false); - CHECK(accept_helper("1.") == false); - CHECK(accept_helper("1E") == false); - CHECK(accept_helper("1E-") == false); - CHECK(accept_helper("1.E1") == false); - CHECK(accept_helper("-1E") == false); - CHECK(accept_helper("-0E#") == false); - CHECK(accept_helper("-0E-#") == false); - CHECK(accept_helper("-0#") == false); - CHECK(accept_helper("-0.0:") == false); - CHECK(accept_helper("-0.0Z") == false); - CHECK(accept_helper("-0E123:") == false); - CHECK(accept_helper("-0e0-:") == false); - CHECK(accept_helper("-0e-:") == false); - CHECK(accept_helper("-0f") == false); - - // numbers must not begin with "+" - CHECK(accept_helper("+1") == false); - CHECK(accept_helper("+0") == false); - } - } - } - - SECTION("parse errors") - { - // unexpected end of number - CHECK_THROWS_WITH_AS(parser_helper("0."), - "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - invalid number; expected digit after '.'; last read: '0.'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("-"), - "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid number; expected digit after '-'; last read: '-'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("--"), - "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid number; expected digit after '-'; last read: '--'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("-0."), - "[json.exception.parse_error.101] parse error at line 1, column 4: syntax error while parsing value - invalid number; expected digit after '.'; last read: '-0.'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("-."), - "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid number; expected digit after '-'; last read: '-.'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("-:"), - "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid number; expected digit after '-'; last read: '-:'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("0.:"), - "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - invalid number; expected digit after '.'; last read: '0.:'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("e."), - "[json.exception.parse_error.101] parse error at line 1, column 1: syntax error while parsing value - invalid literal; last read: 'e'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("1e."), - "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - invalid number; expected '+', '-', or digit after exponent; last read: '1e.'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("1e/"), - "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - invalid number; expected '+', '-', or digit after exponent; last read: '1e/'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("1e:"), - "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - invalid number; expected '+', '-', or digit after exponent; last read: '1e:'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("1E."), - "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - invalid number; expected '+', '-', or digit after exponent; last read: '1E.'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("1E/"), - "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - invalid number; expected '+', '-', or digit after exponent; last read: '1E/'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("1E:"), - "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - invalid number; expected '+', '-', or digit after exponent; last read: '1E:'", json::parse_error&); - - // unexpected end of null - CHECK_THROWS_WITH_AS(parser_helper("n"), - "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid literal; last read: 'n'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("nu"), - "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - invalid literal; last read: 'nu'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("nul"), - "[json.exception.parse_error.101] parse error at line 1, column 4: syntax error while parsing value - invalid literal; last read: 'nul'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("nulk"), - "[json.exception.parse_error.101] parse error at line 1, column 4: syntax error while parsing value - invalid literal; last read: 'nulk'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("nulm"), - "[json.exception.parse_error.101] parse error at line 1, column 4: syntax error while parsing value - invalid literal; last read: 'nulm'", json::parse_error&); - - // unexpected end of true - CHECK_THROWS_WITH_AS(parser_helper("t"), - "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid literal; last read: 't'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("tr"), - "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - invalid literal; last read: 'tr'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("tru"), - "[json.exception.parse_error.101] parse error at line 1, column 4: syntax error while parsing value - invalid literal; last read: 'tru'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("trud"), - "[json.exception.parse_error.101] parse error at line 1, column 4: syntax error while parsing value - invalid literal; last read: 'trud'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("truf"), - "[json.exception.parse_error.101] parse error at line 1, column 4: syntax error while parsing value - invalid literal; last read: 'truf'", json::parse_error&); - - // unexpected end of false - CHECK_THROWS_WITH_AS(parser_helper("f"), - "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid literal; last read: 'f'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("fa"), - "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - invalid literal; last read: 'fa'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("fal"), - "[json.exception.parse_error.101] parse error at line 1, column 4: syntax error while parsing value - invalid literal; last read: 'fal'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("fals"), - "[json.exception.parse_error.101] parse error at line 1, column 5: syntax error while parsing value - invalid literal; last read: 'fals'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("falsd"), - "[json.exception.parse_error.101] parse error at line 1, column 5: syntax error while parsing value - invalid literal; last read: 'falsd'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("falsf"), - "[json.exception.parse_error.101] parse error at line 1, column 5: syntax error while parsing value - invalid literal; last read: 'falsf'", json::parse_error&); - - // missing/unexpected end of array - CHECK_THROWS_WITH_AS(parser_helper("["), - "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - unexpected end of input; expected '[', '{', or a literal", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("[1"), - "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing array - unexpected end of input; expected ']'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("[1,"), - "[json.exception.parse_error.101] parse error at line 1, column 4: syntax error while parsing value - unexpected end of input; expected '[', '{', or a literal", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("[1,]"), - "[json.exception.parse_error.101] parse error at line 1, column 4: syntax error while parsing value - unexpected ']'; expected '[', '{', or a literal", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("]"), - "[json.exception.parse_error.101] parse error at line 1, column 1: syntax error while parsing value - unexpected ']'; expected '[', '{', or a literal", json::parse_error&); - - // missing/unexpected end of object - CHECK_THROWS_WITH_AS(parser_helper("{"), - "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing object key - unexpected end of input; expected string literal", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("{\"foo\""), - "[json.exception.parse_error.101] parse error at line 1, column 7: syntax error while parsing object separator - unexpected end of input; expected ':'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("{\"foo\":"), - "[json.exception.parse_error.101] parse error at line 1, column 8: syntax error while parsing value - unexpected end of input; expected '[', '{', or a literal", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("{\"foo\":}"), - "[json.exception.parse_error.101] parse error at line 1, column 8: syntax error while parsing value - unexpected '}'; expected '[', '{', or a literal", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("{\"foo\":1,}"), - "[json.exception.parse_error.101] parse error at line 1, column 10: syntax error while parsing object key - unexpected '}'; expected string literal", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("}"), - "[json.exception.parse_error.101] parse error at line 1, column 1: syntax error while parsing value - unexpected '}'; expected '[', '{', or a literal", json::parse_error&); - - // missing/unexpected end of string - CHECK_THROWS_WITH_AS(parser_helper("\""), - "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: missing closing quote; last read: '\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\\\""), - "[json.exception.parse_error.101] parse error at line 1, column 4: syntax error while parsing value - invalid string: missing closing quote; last read: '\"\\\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\\u\""), - "[json.exception.parse_error.101] parse error at line 1, column 4: syntax error while parsing value - invalid string: '\\u' must be followed by 4 hex digits; last read: '\"\\u\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\\u0\""), - "[json.exception.parse_error.101] parse error at line 1, column 5: syntax error while parsing value - invalid string: '\\u' must be followed by 4 hex digits; last read: '\"\\u0\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\\u01\""), - "[json.exception.parse_error.101] parse error at line 1, column 6: syntax error while parsing value - invalid string: '\\u' must be followed by 4 hex digits; last read: '\"\\u01\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\\u012\""), - "[json.exception.parse_error.101] parse error at line 1, column 7: syntax error while parsing value - invalid string: '\\u' must be followed by 4 hex digits; last read: '\"\\u012\"'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\\u"), - "[json.exception.parse_error.101] parse error at line 1, column 4: syntax error while parsing value - invalid string: '\\u' must be followed by 4 hex digits; last read: '\"\\u'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\\u0"), - "[json.exception.parse_error.101] parse error at line 1, column 5: syntax error while parsing value - invalid string: '\\u' must be followed by 4 hex digits; last read: '\"\\u0'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\\u01"), - "[json.exception.parse_error.101] parse error at line 1, column 6: syntax error while parsing value - invalid string: '\\u' must be followed by 4 hex digits; last read: '\"\\u01'", json::parse_error&); - CHECK_THROWS_WITH_AS(parser_helper("\"\\u012"), - "[json.exception.parse_error.101] parse error at line 1, column 7: syntax error while parsing value - invalid string: '\\u' must be followed by 4 hex digits; last read: '\"\\u012'", json::parse_error&); - - // invalid escapes - for (int c = 1; c < 128; ++c) - { - auto s = std::string("\"\\") + std::string(1, static_cast(c)) + "\""; - - switch (c) - { - // valid escapes - case ('"'): - case ('\\'): - case ('/'): - case ('b'): - case ('f'): - case ('n'): - case ('r'): - case ('t'): - { - CHECK_NOTHROW(parser_helper(s)); - break; - } - - // \u must be followed with four numbers, so we skip it here - case ('u'): - { - break; - } - - // any other combination of backslash and character is invalid - default: - { - CHECK_THROWS_AS(parser_helper(s), json::parse_error&); - // only check error message if c is not a control character - if (c > 0x1f) - { - CHECK_THROWS_WITH_STD_STR(parser_helper(s), - "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - invalid string: forbidden character after backslash; last read: '\"\\" + std::string(1, static_cast(c)) + "'"); - } - break; - } - } - } - - // invalid \uxxxx escapes - { - // check whether character is a valid hex character - const auto valid = [](int c) - { - switch (c) - { - case ('0'): - case ('1'): - case ('2'): - case ('3'): - case ('4'): - case ('5'): - case ('6'): - case ('7'): - case ('8'): - case ('9'): - case ('a'): - case ('b'): - case ('c'): - case ('d'): - case ('e'): - case ('f'): - case ('A'): - case ('B'): - case ('C'): - case ('D'): - case ('E'): - case ('F'): - { - return true; - } - - default: - { - return false; - } - } - }; - - for (int c = 1; c < 128; ++c) - { - std::string const s = "\"\\u"; - - // create a string with the iterated character at each position - auto s1 = s + "000" + std::string(1, static_cast(c)) + "\""; - auto s2 = s + "00" + std::string(1, static_cast(c)) + "0\""; - auto s3 = s + "0" + std::string(1, static_cast(c)) + "00\""; - auto s4 = s + std::string(1, static_cast(c)) + "000\""; - - if (valid(c)) - { - CAPTURE(s1) - CHECK_NOTHROW(parser_helper(s1)); - CAPTURE(s2) - CHECK_NOTHROW(parser_helper(s2)); - CAPTURE(s3) - CHECK_NOTHROW(parser_helper(s3)); - CAPTURE(s4) - CHECK_NOTHROW(parser_helper(s4)); - } - else - { - CAPTURE(s1) - CHECK_THROWS_AS(parser_helper(s1), json::parse_error&); - // only check error message if c is not a control character - if (c > 0x1f) - { - CHECK_THROWS_WITH_STD_STR(parser_helper(s1), - "[json.exception.parse_error.101] parse error at line 1, column 7: syntax error while parsing value - invalid string: '\\u' must be followed by 4 hex digits; last read: '" + s1.substr(0, 7) + "'"); - } - - CAPTURE(s2) - CHECK_THROWS_AS(parser_helper(s2), json::parse_error&); - // only check error message if c is not a control character - if (c > 0x1f) - { - CHECK_THROWS_WITH_STD_STR(parser_helper(s2), - "[json.exception.parse_error.101] parse error at line 1, column 6: syntax error while parsing value - invalid string: '\\u' must be followed by 4 hex digits; last read: '" + s2.substr(0, 6) + "'"); - } - - CAPTURE(s3) - CHECK_THROWS_AS(parser_helper(s3), json::parse_error&); - // only check error message if c is not a control character - if (c > 0x1f) - { - CHECK_THROWS_WITH_STD_STR(parser_helper(s3), - "[json.exception.parse_error.101] parse error at line 1, column 5: syntax error while parsing value - invalid string: '\\u' must be followed by 4 hex digits; last read: '" + s3.substr(0, 5) + "'"); - } - - CAPTURE(s4) - CHECK_THROWS_AS(parser_helper(s4), json::parse_error&); - // only check error message if c is not a control character - if (c > 0x1f) - { - CHECK_THROWS_WITH_STD_STR(parser_helper(s4), - "[json.exception.parse_error.101] parse error at line 1, column 4: syntax error while parsing value - invalid string: '\\u' must be followed by 4 hex digits; last read: '" + s4.substr(0, 4) + "'"); - } - } - } - } - - json _; - - // missing part of a surrogate pair - CHECK_THROWS_WITH_AS(_ = json::parse("\"\\uD80C\""), "[json.exception.parse_error.101] parse error at line 1, column 8: syntax error while parsing value - invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF; last read: '\"\\uD80C\"'", json::parse_error&); - // invalid surrogate pair - CHECK_THROWS_WITH_AS(_ = json::parse("\"\\uD80C\\uD80C\""), - "[json.exception.parse_error.101] parse error at line 1, column 13: syntax error while parsing value - invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF; last read: '\"\\uD80C\\uD80C'", json::parse_error&); - CHECK_THROWS_WITH_AS(_ = json::parse("\"\\uD80C\\u0000\""), - "[json.exception.parse_error.101] parse error at line 1, column 13: syntax error while parsing value - invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF; last read: '\"\\uD80C\\u0000'", json::parse_error&); - CHECK_THROWS_WITH_AS(_ = json::parse("\"\\uD80C\\uFFFF\""), - "[json.exception.parse_error.101] parse error at line 1, column 13: syntax error while parsing value - invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF; last read: '\"\\uD80C\\uFFFF'", json::parse_error&); - } - - SECTION("parse errors (accept)") - { - // unexpected end of number - CHECK(accept_helper("0.") == false); - CHECK(accept_helper("-") == false); - CHECK(accept_helper("--") == false); - CHECK(accept_helper("-0.") == false); - CHECK(accept_helper("-.") == false); - CHECK(accept_helper("-:") == false); - CHECK(accept_helper("0.:") == false); - CHECK(accept_helper("e.") == false); - CHECK(accept_helper("1e.") == false); - CHECK(accept_helper("1e/") == false); - CHECK(accept_helper("1e:") == false); - CHECK(accept_helper("1E.") == false); - CHECK(accept_helper("1E/") == false); - CHECK(accept_helper("1E:") == false); - - // unexpected end of null - CHECK(accept_helper("n") == false); - CHECK(accept_helper("nu") == false); - CHECK(accept_helper("nul") == false); - - // unexpected end of true - CHECK(accept_helper("t") == false); - CHECK(accept_helper("tr") == false); - CHECK(accept_helper("tru") == false); - - // unexpected end of false - CHECK(accept_helper("f") == false); - CHECK(accept_helper("fa") == false); - CHECK(accept_helper("fal") == false); - CHECK(accept_helper("fals") == false); - - // missing/unexpected end of array - CHECK(accept_helper("[") == false); - CHECK(accept_helper("[1") == false); - CHECK(accept_helper("[1,") == false); - CHECK(accept_helper("[1,]") == false); - CHECK(accept_helper("]") == false); - - // missing/unexpected end of object - CHECK(accept_helper("{") == false); - CHECK(accept_helper("{\"foo\"") == false); - CHECK(accept_helper("{\"foo\":") == false); - CHECK(accept_helper("{\"foo\":}") == false); - CHECK(accept_helper("{\"foo\":1,}") == false); - CHECK(accept_helper("}") == false); - - // missing/unexpected end of string - CHECK(accept_helper("\"") == false); - CHECK(accept_helper("\"\\\"") == false); - CHECK(accept_helper("\"\\u\"") == false); - CHECK(accept_helper("\"\\u0\"") == false); - CHECK(accept_helper("\"\\u01\"") == false); - CHECK(accept_helper("\"\\u012\"") == false); - CHECK(accept_helper("\"\\u") == false); - CHECK(accept_helper("\"\\u0") == false); - CHECK(accept_helper("\"\\u01") == false); - CHECK(accept_helper("\"\\u012") == false); - - // unget of newline - CHECK(parser_helper("\n123\n") == 123); - - // invalid escapes - for (int c = 1; c < 128; ++c) - { - auto s = std::string("\"\\") + std::string(1, static_cast(c)) + "\""; - - switch (c) - { - // valid escapes - case ('"'): - case ('\\'): - case ('/'): - case ('b'): - case ('f'): - case ('n'): - case ('r'): - case ('t'): - { - CHECK(json::parser(nlohmann::detail::input_adapter(s)).accept()); - break; - } - - // \u must be followed with four numbers, so we skip it here - case ('u'): - { - break; - } - - // any other combination of backslash and character is invalid - default: - { - CHECK(json::parser(nlohmann::detail::input_adapter(s)).accept() == false); - break; - } - } - } - - // invalid \uxxxx escapes - { - // check whether character is a valid hex character - const auto valid = [](int c) - { - switch (c) - { - case ('0'): - case ('1'): - case ('2'): - case ('3'): - case ('4'): - case ('5'): - case ('6'): - case ('7'): - case ('8'): - case ('9'): - case ('a'): - case ('b'): - case ('c'): - case ('d'): - case ('e'): - case ('f'): - case ('A'): - case ('B'): - case ('C'): - case ('D'): - case ('E'): - case ('F'): - { - return true; - } - - default: - { - return false; - } - } - }; - - for (int c = 1; c < 128; ++c) - { - std::string const s = "\"\\u"; - - // create a string with the iterated character at each position - const auto s1 = s + "000" + std::string(1, static_cast(c)) + "\""; - const auto s2 = s + "00" + std::string(1, static_cast(c)) + "0\""; - const auto s3 = s + "0" + std::string(1, static_cast(c)) + "00\""; - const auto s4 = s + std::string(1, static_cast(c)) + "000\""; - - if (valid(c)) - { - CAPTURE(s1) - CHECK(json::parser(nlohmann::detail::input_adapter(s1)).accept()); - CAPTURE(s2) - CHECK(json::parser(nlohmann::detail::input_adapter(s2)).accept()); - CAPTURE(s3) - CHECK(json::parser(nlohmann::detail::input_adapter(s3)).accept()); - CAPTURE(s4) - CHECK(json::parser(nlohmann::detail::input_adapter(s4)).accept()); - } - else - { - CAPTURE(s1) - CHECK(json::parser(nlohmann::detail::input_adapter(s1)).accept() == false); - - CAPTURE(s2) - CHECK(json::parser(nlohmann::detail::input_adapter(s2)).accept() == false); - - CAPTURE(s3) - CHECK(json::parser(nlohmann::detail::input_adapter(s3)).accept() == false); - - CAPTURE(s4) - CHECK(json::parser(nlohmann::detail::input_adapter(s4)).accept() == false); - } - } - } - - // missing part of a surrogate pair - CHECK(accept_helper("\"\\uD80C\"") == false); - // invalid surrogate pair - CHECK(accept_helper("\"\\uD80C\\uD80C\"") == false); - CHECK(accept_helper("\"\\uD80C\\u0000\"") == false); - CHECK(accept_helper("\"\\uD80C\\uFFFF\"") == false); - } - - SECTION("tests found by mutate++") - { - // test case to make sure no comma precedes the first key - CHECK_THROWS_WITH_AS(parser_helper("{,\"key\": false}"), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing object key - unexpected ','; expected string literal", json::parse_error&); - // test case to make sure an object is properly closed - CHECK_THROWS_WITH_AS(parser_helper("[{\"key\": false true]"), "[json.exception.parse_error.101] parse error at line 1, column 19: syntax error while parsing object - unexpected true literal; expected '}'", json::parse_error&); - - // test case to make sure the callback is properly evaluated after reading a key - { - json::parser_callback_t const cb = [](int /*unused*/, json::parse_event_t event, json& /*unused*/) noexcept - { - return event != json::parse_event_t::key; - }; - - const json x = json::parse("{\"key\": false}", cb); - CHECK(x == json::object()); - } - } - - SECTION("callback function") - { - const auto* s_object = R"( - { - "foo": 2, - "bar": { - "baz": 1 - } - } - )"; - - const auto* s_array = R"( - [1,2,[3,4,5],4,5] - )"; - - const auto* structured_array = R"( - [ - 1, - { - "foo": "bar" - }, - { - "qux": "baz" - } - ] - )"; - - SECTION("filter nothing") - { - const json j_object = json::parse(s_object, [](int /*unused*/, json::parse_event_t /*unused*/, const json& /*unused*/) noexcept - { - return true; - }); - - CHECK (j_object == json({{"foo", 2}, {"bar", {{"baz", 1}}}})); - - const json j_array = json::parse(s_array, [](int /*unused*/, json::parse_event_t /*unused*/, const json& /*unused*/) noexcept - { - return true; - }); - - CHECK (j_array == json({1, 2, {3, 4, 5}, 4, 5})); - } - - SECTION("filter everything") - { - json const j_object = json::parse(s_object, [](int /*unused*/, json::parse_event_t /*unused*/, const json& /*unused*/) noexcept - { - return false; - }); - - // the top-level object will be discarded, leaving a null - CHECK (j_object.is_null()); - - json const j_array = json::parse(s_array, [](int /*unused*/, json::parse_event_t /*unused*/, const json& /*unused*/) noexcept - { - return false; - }); - - // the top-level array will be discarded, leaving a null - CHECK (j_array.is_null()); - } - - SECTION("filter specific element") - { - const json j_object = json::parse(s_object, [](int /*unused*/, json::parse_event_t event, const json & j) noexcept - { - // filter all number(2) elements - return event != json::parse_event_t::value || j != json(2); - }); - - CHECK (j_object == json({{"bar", {{"baz", 1}}}})); - - const json j_array = json::parse(s_array, [](int /*unused*/, json::parse_event_t event, const json & j) noexcept - { - return event != json::parse_event_t::value || j != json(2); - }); - - CHECK (j_array == json({1, {3, 4, 5}, 4, 5})); - } - - SECTION("filter object in array") - { - const json j_filtered1 = json::parse(structured_array, [](int /*unused*/, json::parse_event_t e, const json & parsed) - { - return !(e == json::parse_event_t::object_end && parsed.contains("foo")); - }); - - // the specified object will be discarded, and removed. - CHECK (j_filtered1.size() == 2); - CHECK (j_filtered1 == json({1, {{"qux", "baz"}}})); - - const json j_filtered2 = json::parse(structured_array, [](int /*unused*/, json::parse_event_t e, const json& /*parsed*/) noexcept - { - return e != json::parse_event_t::object_end; - }); - - // removed all objects in array. - CHECK (j_filtered2.size() == 1); - CHECK (j_filtered2 == json({1})); - } - - SECTION("filter specific events") - { - SECTION("first closing event") - { - { - const json j_object = json::parse(s_object, [](int /*unused*/, json::parse_event_t e, const json& /*unused*/) noexcept - { - static bool first = true; - if (e == json::parse_event_t::object_end && first) - { - first = false; - return false; - } - - return true; - }); - - // the first completed object will be discarded - CHECK (j_object == json({{"foo", 2}})); - } - - { - const json j_array = json::parse(s_array, [](int /*unused*/, json::parse_event_t e, const json& /*unused*/) noexcept - { - static bool first = true; - if (e == json::parse_event_t::array_end && first) - { - first = false; - return false; - } - - return true; - }); - - // the first completed array will be discarded - CHECK (j_array == json({1, 2, 4, 5})); - } - } - } - - SECTION("special cases") - { - // the following test cases cover the situation in which an empty - // object and array is discarded only after the closing character - // has been read - - const json j_empty_object = json::parse("{}", [](int /*unused*/, json::parse_event_t e, const json& /*unused*/) noexcept - { - return e != json::parse_event_t::object_end; - }); - CHECK(j_empty_object == json()); - - const json j_empty_array = json::parse("[]", [](int /*unused*/, json::parse_event_t e, const json& /*unused*/) noexcept - { - return e != json::parse_event_t::array_end; - }); - CHECK(j_empty_array == json()); - } - } - - SECTION("constructing from contiguous containers") - { - SECTION("from std::vector") - { - std::vector v = {'t', 'r', 'u', 'e'}; - json j; - json::parser(nlohmann::detail::input_adapter(std::begin(v), std::end(v))).parse(true, j); - CHECK(j == json(true)); - } - - SECTION("from std::array") - { - std::array v { {'t', 'r', 'u', 'e'} }; - json j; - json::parser(nlohmann::detail::input_adapter(std::begin(v), std::end(v))).parse(true, j); - CHECK(j == json(true)); - } - - SECTION("from array") - { - uint8_t v[] = {'t', 'r', 'u', 'e'}; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) - json j; - json::parser(nlohmann::detail::input_adapter(std::begin(v), std::end(v))).parse(true, j); - CHECK(j == json(true)); - } - - SECTION("from char literal") - { - CHECK(parser_helper("true") == json(true)); - } - - SECTION("from std::string") - { - std::string v = {'t', 'r', 'u', 'e'}; - json j; - json::parser(nlohmann::detail::input_adapter(std::begin(v), std::end(v))).parse(true, j); - CHECK(j == json(true)); - } - - SECTION("from std::initializer_list") - { - std::initializer_list const v = {'t', 'r', 'u', 'e'}; - json j; - json::parser(nlohmann::detail::input_adapter(std::begin(v), std::end(v))).parse(true, j); - CHECK(j == json(true)); - } - - SECTION("from std::valarray") - { - std::valarray v = {'t', 'r', 'u', 'e'}; - json j; - json::parser(nlohmann::detail::input_adapter(std::begin(v), std::end(v))).parse(true, j); - CHECK(j == json(true)); - } - } - - SECTION("improve test coverage") - { - SECTION("parser with callback") - { - json::parser_callback_t const cb = [](int /*unused*/, json::parse_event_t /*unused*/, json& /*unused*/) noexcept - { - return true; - }; - - CHECK(json::parse("{\"foo\": true:", cb, false).is_discarded()); - - json _; - CHECK_THROWS_WITH_AS(_ = json::parse("{\"foo\": true:", cb), "[json.exception.parse_error.101] parse error at line 1, column 13: syntax error while parsing object - unexpected ':'; expected '}'", json::parse_error&); - - CHECK_THROWS_WITH_AS(_ = json::parse("1.18973e+4932", cb), "[json.exception.out_of_range.406] number overflow parsing '1.18973e+4932'", json::out_of_range&); - } - - SECTION("SAX parser") - { - SECTION("} without value") - { - SaxCountdown s(1); - CHECK(json::sax_parse("{}", &s) == false); - } - - SECTION("} with value") - { - SaxCountdown s(3); - CHECK(json::sax_parse("{\"k1\": true}", &s) == false); - } - - SECTION("second key") - { - SaxCountdown s(3); - CHECK(json::sax_parse("{\"k1\": true, \"k2\": false}", &s) == false); - } - - SECTION("] without value") - { - SaxCountdown s(1); - CHECK(json::sax_parse("[]", &s) == false); - } - - SECTION("] with value") - { - SaxCountdown s(2); - CHECK(json::sax_parse("[1]", &s) == false); - } - - SECTION("float") - { - SaxCountdown s(0); - CHECK(json::sax_parse("3.14", &s) == false); - } - - SECTION("false") - { - SaxCountdown s(0); - CHECK(json::sax_parse("false", &s) == false); - } - - SECTION("null") - { - SaxCountdown s(0); - CHECK(json::sax_parse("null", &s) == false); - } - - SECTION("true") - { - SaxCountdown s(0); - CHECK(json::sax_parse("true", &s) == false); - } - - SECTION("unsigned") - { - SaxCountdown s(0); - CHECK(json::sax_parse("12", &s) == false); - } - - SECTION("integer") - { - SaxCountdown s(0); - CHECK(json::sax_parse("-12", &s) == false); - } - - SECTION("string") - { - SaxCountdown s(0); - CHECK(json::sax_parse("\"foo\"", &s) == false); - } - } - } - - SECTION("error messages for comments") - { - json _; - CHECK_THROWS_WITH_AS(_ = json::parse("/a", nullptr, true, true), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid comment; expecting '/' or '*' after '/'; last read: '/a'", json::parse_error); - CHECK_THROWS_WITH_AS(_ = json::parse("/*", nullptr, true, true), "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - invalid comment; missing closing '*/'; last read: '/*'", json::parse_error); - } - - // Macro for all test cases for start_pos and end_pos -#define SETUP_TESTCASES() \ - SECTION("with callback") \ - { \ - SECTION("filter nothing") \ - { \ - json::parser_callback_t const cb = [](int /*unused*/, json::parse_event_t /*unused*/, json& /*unused*/) noexcept \ - { \ - return true; \ - }; \ - validate_start_end_pos_for_nested_obj_helper(nested_type_json_str, root_type_json_str, expected, cb); \ - } \ - SECTION("filter element") \ - { \ - json::parser_callback_t const cb = [](int /*unused*/, json::parse_event_t event, json& j) noexcept \ - { \ - return (event != json::parse_event_t::key && event != json::parse_event_t::value) || j != json("a"); \ - }; \ - validate_start_end_pos_for_nested_obj_helper(nested_type_json_str, root_type_json_str, filteredExpected, cb); \ - } \ - } \ - SECTION("without callback") \ - { \ - validate_start_end_pos_for_nested_obj_helper(nested_type_json_str, root_type_json_str, expected); \ - } - - SECTION("retrieve start position and end position") - { - SECTION("for object") - { - // Create an object with spaces to test the start and end positions. Spaces will not be included in the - // JSON object, however, the start and end positions should include the spaces from the input JSON string. - const std::string nested_type_json_str = R"({ "a": 1,"b" : "test1"})"; - const std::string root_type_json_str = R"({ "nested": )" + nested_type_json_str + R"(, "anotherValue": "test2"})"; - auto expected = json({{"nested", {{"a", 1}, {"b", "test1"}}}, {"anotherValue", "test2"}}); - auto filteredExpected = expected; - filteredExpected["nested"].erase("a"); - - SETUP_TESTCASES() - } - - SECTION("for array") - { - const std::string nested_type_json_str = R"(["a", "test", 45])"; - const std::string root_type_json_str = R"({ "nested": )" + nested_type_json_str + R"(, "anotherValue": "test" })"; - auto expected = json({{"nested", {"a", "test", 45}}, {"anotherValue", "test"}}); - auto filteredExpected = expected; - filteredExpected["nested"] = json({"test", 45}); - SETUP_TESTCASES() - } - - SECTION("for array with objects") - { - const std::string nested_type_json_str = R"([{"a": 1, "b": "test"}, {"c": 2, "d": "test2"}])"; - const std::string root_type_json_str = R"({ "nested": )" + nested_type_json_str + R"(, "anotherValue": "test" })"; - auto expected = json({{"nested", {{{"a", 1}, {"b", "test"}}, {{"c", 2}, {"d", "test2"}}}}, {"anotherValue", "test"}}); - auto filteredExpected = expected; - filteredExpected["nested"][0].erase("a"); - SETUP_TESTCASES() - - auto j = json::parse(root_type_json_str); - auto nested_array = j["nested"]; - const auto& nested_obj = nested_array[0]; - CHECK(nested_type_json_str.substr(1, 21) == root_type_json_str.substr(nested_obj.start_pos(), nested_obj.end_pos() - nested_obj.start_pos())); - CHECK(nested_type_json_str.substr(24, 22) == root_type_json_str.substr(nested_array[1].start_pos(), nested_array[1].end_pos() - nested_array[1].start_pos())); - } - - SECTION("for two levels of nesting objects") - { - const std::string nested_type_json_str = R"({"nested2": {"b": "test"}})"; - const std::string root_type_json_str = R"({ "a": 2, "nested": )" + nested_type_json_str + R"(, "anotherValue": "test" })"; - auto expected = json({{"a", 2}, {"nested", {{"nested2", {{"b", "test"}}}}}, {"anotherValue", "test"}}); - auto filteredExpected = expected; - filteredExpected.erase("a"); - SETUP_TESTCASES() - - auto j = json::parse(root_type_json_str); - auto nested_obj = j["nested"]["nested2"]; - CHECK(nested_type_json_str.substr(12, 13) == root_type_json_str.substr(nested_obj.start_pos(), nested_obj.end_pos() - nested_obj.start_pos())); - } - - SECTION("for simple types") - { - SECTION("no nested") - { - SECTION("with callback") - { - json::parser_callback_t const cb = [](int /*unused*/, json::parse_event_t /*unused*/, json& /*unused*/) noexcept - { - return true; - }; - - // 1. string type - std::string json_str = R"("test")"; - auto j = json::parse(json_str, cb); - validate_generated_json_and_start_end_pos_helper(json_str, j, "test"); - - // 2. number type - json_str = R"(1)"; - j = json::parse(json_str, cb); - validate_generated_json_and_start_end_pos_helper(json_str, j, 1); - - // 3. boolean type - json_str = R"(true)"; - j = json::parse(json_str, cb); - validate_generated_json_and_start_end_pos_helper(json_str, j, true); - - // 4. null type - json_str = R"(null)"; - j = json::parse(json_str, cb); - validate_generated_json_and_start_end_pos_helper(json_str, j, nullptr); - } - - SECTION("without callback") - { - // 1. string type - std::string json_str = R"("test")"; - auto j = json::parse(json_str); - validate_generated_json_and_start_end_pos_helper(json_str, j, "test"); - - // 2. number type - json_str = R"(1)"; - j = json::parse(json_str); - validate_generated_json_and_start_end_pos_helper(json_str, j, 1); - - json_str = R"(1.001239923)"; - j = json::parse(json_str); - validate_generated_json_and_start_end_pos_helper(json_str, j, 1.001239923); - - json_str = R"(1.123812389000000)"; - j = json::parse(json_str); - validate_generated_json_and_start_end_pos_helper(json_str, j, 1.123812389); - - // 3. boolean type - json_str = R"(true)"; - j = json::parse(json_str); - validate_generated_json_and_start_end_pos_helper(json_str, j, true); - - json_str = R"(false)"; - j = json::parse(json_str); - validate_generated_json_and_start_end_pos_helper(json_str, j, false); - - // 4. null type - json_str = R"(null)"; - j = json::parse(json_str); - validate_generated_json_and_start_end_pos_helper(json_str, j, nullptr); - } - } - - SECTION("string type") - { - const std::string nested_type_json_str = R"("test")"; - const std::string root_type_json_str = R"({ "a": 1, "nested": )" + nested_type_json_str + R"(, "anotherValue": "test" })"; - auto expected = json({{"nested", "test"}, {"anotherValue", "test"}, {"a", 1}}); - auto filteredExpected = expected; - filteredExpected.erase("a"); - SETUP_TESTCASES() - } - - SECTION("number type") - { - const std::string nested_type_json_str = R"(2)"; - const std::string root_type_json_str = R"({ "a": 1, "nested": )" + nested_type_json_str + R"(, "anotherValue": "test" })"; - auto expected = json({{"nested", 2}, {"anotherValue", "test"}, {"a", 1}}); - auto filteredExpected = expected; - filteredExpected.erase("a"); - SETUP_TESTCASES() - } - - SECTION("boolean type") - { - const std::string nested_type_json_str = R"(true)"; - const std::string root_type_json_str = R"({ "a": 1, "nested": )" + nested_type_json_str + R"(, "anotherValue": "test" })"; - auto expected = json({{"nested", true}, {"anotherValue", "test"}, {"a", 1}}); - auto filteredExpected = expected; - filteredExpected.erase("a"); - SETUP_TESTCASES() - } - - SECTION("null type") - { - const std::string nested_type_json_str = R"(null)"; - const std::string root_type_json_str = R"({ "a": 1, "nested": )" + nested_type_json_str + R"(, "anotherValue": "test" })"; - auto expected = json({{"nested", nullptr}, {"anotherValue", "test"}, {"a", 1}}); - auto filteredExpected = expected; - filteredExpected.erase("a"); - SETUP_TESTCASES() - } - } - SECTION("with leading whitespace and newlines around root JSON") - { - const std::string initial_whitespace = R"( - - )"; - const std::string nested_type_json_str = R"({ - "a": 1, - "nested": { - "b": "test" - }, - "anotherValue": "test" - })"; - const std::string end_whitespace = R"( - - )"; - const std::string root_type_json_str = initial_whitespace + nested_type_json_str + end_whitespace; - - auto expected = json({{"a", 1}, {"nested", {{"b", "test"}}}, {"anotherValue", "test"}}); - - auto j = json::parse(root_type_json_str); - - // 2. Check if the generated JSON is as expected - CHECK(j == expected); - - // 3. Check if the start and end positions do not include the surrounding whitespace - CHECK(j.start_pos() == initial_whitespace.size()); - CHECK(j.end_pos() == root_type_json_str.size() - end_whitespace.size()); - } - } -} diff --git a/tests/src/unit-diagnostic-positions-only.cpp b/tests/src/unit-diagnostic-positions-only.cpp deleted file mode 100644 index 735376514..000000000 --- a/tests/src/unit-diagnostic-positions-only.cpp +++ /dev/null @@ -1,44 +0,0 @@ -// __ _____ _____ _____ -// __| | __| | | | JSON for Modern C++ (supporting code) -// | | |__ | | | | | | version 3.12.0 -// |_____|_____|_____|_|___| https://github.com/nlohmann/json -// -// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann -// SPDX-License-Identifier: MIT - -#include "doctest_compatibility.h" - -#ifdef JSON_DIAGNOSTICS - #undef JSON_DIAGNOSTICS -#endif - -#define JSON_DIAGNOSTICS 0 -#define JSON_DIAGNOSTIC_POSITIONS 1 -#include - -using json = nlohmann::json; - -TEST_CASE("Better diagnostics with positions only") -{ - SECTION("invalid type") - { - const std::string json_invalid_string = R"( - { - "address": { - "street": "Fake Street", - "housenumber": "1" - } - } - )"; - json j = json::parse(json_invalid_string); - CHECK_THROWS_WITH_AS(j.at("address").at("housenumber").get(), - "[json.exception.type_error.302] (bytes 108-111) type must be number, but is string", json::type_error); - } - - SECTION("invalid type without positions") - { - const json j = "foo"; - CHECK_THROWS_WITH_AS(j.get(), - "[json.exception.type_error.302] type must be number, but is string", json::type_error); - } -} diff --git a/tests/src/unit-diagnostic-positions.cpp b/tests/src/unit-diagnostic-positions.cpp index ad9527540..4d2f50a98 100644 --- a/tests/src/unit-diagnostic-positions.cpp +++ b/tests/src/unit-diagnostic-positions.cpp @@ -8,7 +8,9 @@ #include "doctest_compatibility.h" -#define JSON_DIAGNOSTICS 1 +#ifndef JSON_DIAGNOSTICS + #define JSON_DIAGNOSTICS 1 +#endif #define JSON_DIAGNOSTIC_POSITIONS 1 #include @@ -27,8 +29,13 @@ TEST_CASE("Better diagnostics with positions") } )"; json j = json::parse(json_invalid_string); +#if JSON_DIAGNOSTICS CHECK_THROWS_WITH_AS(j.at("address").at("housenumber").get(), "[json.exception.type_error.302] (/address/housenumber) (bytes 108-111) type must be number, but is string", json::type_error); +#else + CHECK_THROWS_WITH_AS(j.at("address").at("housenumber").get(), + "[json.exception.type_error.302] (bytes 108-111) type must be number, but is string", json::type_error); +#endif } SECTION("invalid type without positions") @@ -74,7 +81,12 @@ TEST_CASE("Better diagnostics with positions") // (/foo/bar); the position of that parent is reported in the message const json doc = json::parse(R"({"foo":{"bar":"a string"}})"); const json patch = json::parse(R"([{"op":"add","path":"/foo/bar/baz","value":1}])"); +#if JSON_DIAGNOSTICS CHECK_THROWS_WITH_AS(doc.patch(patch), "[json.exception.out_of_range.411] (/foo/bar) (bytes 14-24) cannot add value: the JSON Patch 'add' target's parent is of type string, but must be an object or array", json::out_of_range); +#else + CHECK_THROWS_WITH_AS(doc.patch(patch), + "[json.exception.out_of_range.411] (bytes 14-24) cannot add value: the JSON Patch 'add' target's parent is of type string, but must be an object or array", json::out_of_range); +#endif } } From ff80ed329564c51b8b7e1203db2b114ce7815b42 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Fri, 11 Sep 2026 08:22:49 +0200 Subject: [PATCH 06/18] Speed up dump(), and keep it from overflowing the stack (#5285) * Add SWAR bulk fast path to string serialization (dump_escaped) When ensure_ascii is false, dump_escaped previously ran every byte of every string and object key through the UTF-8 DFA decoder, even for the common case of ordinary text with nothing to escape. This mirrors the per-byte cost the parser had before the contiguous fast paths. At a character boundary, bulk-copy the longest run of bytes that need no escaping using string_bulk_run() - the same SWAR scanner and UTF-8 bulk validator the lexer's contiguous path uses - and only fall back to the byte-at-a-time DFA loop for the first byte that needs individual handling (a quote, backslash, control character, or ill-formed/truncated UTF-8). Because every "hard" or invalid byte is still processed by the unchanged byte path, escaping output and error handling (including strict-mode error 316 position and message) are byte-identical to before. The ensure_ascii=true path is unchanged: it must escape non-ASCII and 0x7F, which string_bulk_run does not stop on, so a separate predicate would be needed for it. Verified byte-for-byte identical dump output against the pre-change implementation across ~20k randomized byte strings plus curated edge cases (all escapes, control chars, valid multibyte, surrogates, overlong, truncated sequences) for both ensure_ascii settings and all three error handlers, in C++11/17/20 at -O2/-O3. Throughput (g++ -O3, ensure_ascii=false, vs pre-change): long ASCII strings 4.2x twitter-like objects 2.3x dense CJK 1.4x (further headroom with JSON_USE_SIMDUTF) Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01XAYM1qhSA2FDaDcGfPW3fG Signed-off-by: Niels Lohmann * Buffer serializer output and add ensure_ascii string fast path Two further serialization speedups on top of the ensure_ascii=false bulk copy, both reusing the SWAR primitives in detail/input/string_scan.hpp. 1. Internal write buffer (devirtualization). Every structural character ('{', '"', ',', ...) previously went straight to the output adapter through a virtual call. Route all writes through put_char/put_chars into a 1 KiB buffer that flushes in bulk; the public dump() flushes once the top-level value is done (the recursive worker is split out as dump_internal). Runs larger than the buffer are written straight through, so large payloads are not copied twice. This is the dominant cost for object/array-heavy values. 2. ensure_ascii fast path. dump_escaped previously ran the UTF-8 DFA over every byte when escaping non-ASCII. Add find_ascii_copyable_run() (a SWAR scan stopping at '"', '\\', < 0x20, 0x7F, and >= 0x80) so runs of printable ASCII are bulk-copied, with the byte path handling each escape/non-ASCII byte exactly as before. Behavior is unchanged: dump output is byte-for-byte identical to the previous implementation across ~20k randomized byte strings plus curated edge cases (all escapes, control chars, 0x7F, valid multibyte, surrogates, overlong, truncated), for object/array/pretty output, both ensure_ascii settings, and all three error handlers, in C++11/17/20 at -O2/-O3. New unit tests cover the buffer flush boundaries, the escape and 0x7F handling, multibyte under both settings, and invalid-UTF-8 handling. Throughput (g++ -O3, vs the ensure_ascii=false-only baseline): long ASCII, ensure_ascii=0 4.2x long ASCII, ensure_ascii=1 4.1x twitter-like objects 2.7x dense CJK 1.8x Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01XAYM1qhSA2FDaDcGfPW3fG Signed-off-by: Niels Lohmann * Flush serializer buffer in dump_escaped unit test test-convenience failed (macOS finished first; the failure is platform-independent) because check_escaped() calls the internal serializer::dump_escaped() directly and then reads the output stream. Since dump_escaped() now writes into the serializer's internal write buffer, the bytes were still buffered and the stream was empty. Expose flush() under JSON_PRIVATE_UNLESS_TESTED (same visibility as dump_escaped) and flush in check_escaped() before inspecting the output. Per-string flushing inside dump_escaped() was rejected on purpose: it would defeat the buffering that makes object/array-heavy dumps faster. Library behavior is unchanged (flush()'s body is identical; only its access label moved). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01XAYM1qhSA2FDaDcGfPW3fG Signed-off-by: Niels Lohmann * Avoid deep recursion in serialization write-buffer test The "many small structural writes exceed the write buffer" subcase built a 1100-deep nested array and dumped it to force >1024 consecutive single-character writes through put_char (exercising the write buffer's flush-when-full branch). dump() recurses per nesting level, so on MSVC debug builds (smaller default stack, larger frames) this overflowed the stack and crashed test-serialization; Linux/macOS have enough headroom to hide it. Replace the nesting with a flat array of 500 empty strings. Each element emits '"', '"', ',' via put_char, so the dump is a long run of single-character writes (1501 bytes > the 1024-byte buffer) at nesting depth two, hitting the same flush branch without deep recursion. Library code is unchanged. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01XAYM1qhSA2FDaDcGfPW3fG Signed-off-by: Niels Lohmann * Split the write-buffer helpers and write indentation directly Follow-up to @gregmarr's review: put_chars() was doing four unrelated jobs, so give the two that can be made safe their own entry points. - put_literal(): takes the literal by reference and deduces the length from the array bound, so the 27 hand-counted lengths at the call sites can no longer drift from the literals they describe. A literal is checked at compile time to fit the buffer, so this path needs no write-through branch. - put_buffer(): takes the fixed-size buffer itself rather than a bare pointer, so the length can be checked against the buffer's own bound. - put_indent(): memsets the indentation into the write buffer, filling and flushing it as needed. This removes indent_string entirely, and with it both bugs of #5186: the indentation string was grown by doubling, which is not enough when indent_step more than doubles it (a heap over-read - dump(2000) read 2000 bytes out of a 1024-byte string), and the grown part was filled with a space instead of the configured indent_char. next_indent() keeps that PR's assertion against the unsigned indentation accumulation wrapping on deep nesting. put_chars() keeps the two cases that are genuinely a pointer and a count: the run-length copies out of the string being escaped, and to_chars() output. Tests cover an indent_step wider than the write buffer, a non-space indentation character past the old growth point, and nesting whose accumulated indentation spans several buffer-fulls. All three fail against develop. Signed-off-by: Niels Lohmann * Fill the indentation buffer once instead of once per flush @gregmarr's point on the fill-and-flush loop: flushing does not disturb what the write buffer holds, so an indentation spanning several buffer-fulls only has to be written into the buffer once and can then be handed to the adapter as many times as needed. The loop re-filled it every time, doing work it already knew was there. put_indent() now fills the room left in the buffer, and if anything remains, flushes, fills the buffer once, and re-flushes that same content. It also returns early for a zero-width indentation, which is what the closing brace of every outermost value asks for. Measured over a dump(), counting memset calls and bytes inside put_indent: indent before after 4 1 call / 4 B 1 call / 4 B 2000 2 calls / 2000 B 2 calls / 2046 B 100000 98 calls / 100000 B 2 calls / 2046 B The wide case is now constant work rather than proportional to the indentation width; ordinary widths are unchanged. Tests extended to cover several whole buffer-fulls and an exact multiple of the buffer size. Signed-off-by: Niels Lohmann * Tighten the write-buffer helpers after review More of @gregmarr's review on the put_* split: - Reattach the put_chars() doc comment, which the new helpers had been inserted in front of, leaving it describing put_indent(). - Compute the literal length once in put_literal() instead of spelling N - 1 at each use. - Add put_string(str, start, end), which keeps the pointer arithmetic and the bounds assertions inside the function instead of at the call site. With dump_float()'s to_chars() output moved onto put_buffer() as well, put_chars() now has no callers outside put_string()/put_buffer(): nothing passes a bare pointer and a count any more. - Carry the indentation as std::size_t rather than unsigned int. It is a size, it is compared and combined with buffer sizes throughout, and the casts in put_indent() disappear. next_indent() keeps its assertion, which is far harder to trip on a 64-bit size_t but still reachable where that is 32 bits. No output change: pretty and compact dumps, binary values included, are byte-identical to develop. Signed-off-by: Niels Lohmann * Silence avoid-c-arrays on put_literal's array reference clang-tidy flags the reference-to-array parameter under cppcoreguidelines/hicpp/modernize-avoid-c-arrays, and the CI treats warnings as errors. Binding to the array is the whole point here - it is what lets the length be deduced from the literal instead of hand-written at the call site - so suppress it the same way from_json(), to_json() and get_to() already suppress it for their own T (&arr)[N] parameters. Signed-off-by: Niels Lohmann * Bound the descent of dump() Serializing a container serializes its elements, so dump() descended into one call per nesting level. A value nested deeply enough exhausted the call stack and terminated the process with a segmentation fault - no exception, nothing the caller could catch. Parsing such a value works, as the parser is iterative, and so does destroying one, as #1436 made destruction iterative. Bound how far the descent goes rather than take the call stack away from it. The first 128 levels are written by exactly the code that always wrote them, and only below that does dump_iteratively write out what is left, keeping the containers it has entered on an explicit stack. Serializing can therefore no longer exhaust the stack, however deeply a value is nested, while a value nested less deeply than the bound pays only for one comparison per container. Writing every value that way instead measured between 2% and 20% slower - 20% on object-heavy documents - which is why the descent is kept for all but the values that cannot afford it. The bound costs nothing measurable: between -1.4% and +1.2% across compact and pretty output of number, integer, string, object-heavy, wide-object and deeply nested documents. The output is unchanged for every value. Both ways of writing a container emit the separator in front of every element but the first, rather than after every element but the last, which puts exactly one between each pair and none at the end. This fixes #5387 for dump(). The copy constructor is fixed in #5389. Signed-off-by: Niels Lohmann * Fold ensure_ascii into the escaper and write bytes without dump_integer Two hot spots that the write buffer and the bulk scanner left behind. dump_escaped took ensure_ascii as a runtime flag and tested it inside the loop, once per character run, although it cannot change while a string is written. It is now a template parameter, dispatched once per string, which folds the choice of scanner and lets each of the two be inlined into a loop of its own. This is the hottest loop in the serializer: it runs over every string and every object key. A binary value's bytes went through dump_integer, which counts digits and does 64-bit arithmetic for a number that is always in [0, 255]. dump_byte writes the three digits it takes at most straight into the write buffer instead. Any byte type that is not a plain unsigned byte is still left to dump_integer, whose representation of it may differ. Measured against the previous commit (medians of 9 interleaved runs, clang -O3): binary values -33.8%, dense CJK with ensure_ascii -20.6%, key-heavy objects -17.8%, deeply nested pretty output -17.9%, dense CJK without ensure_ascii -11.8%, object-heavy documents -9.3% compact and -9.5% pretty, a small value dumped in a loop -21.4%, wide objects -2.3%. Arrays of plain ASCII strings measured 3.5% to 4.2% slower, the one shape that loses; number and integer arrays are unchanged. Also tried and dropped: leaving the write and string buffers uninitialized rather than zeroing 1.5 KB per dump() call. It is worth -30% on small values, but two nearly identical string workloads moved 18% apart in opposite directions, so the measurements did not support it. The output is unchanged for every value: the differential now also covers every one of the 256 byte values, alone and together, in both binary layouts. Signed-off-by: Niels Lohmann * Write a byte without walking a pointer over the buffer clang-tidy's misc-const-correctness reads the pointer dump_byte advanced over the write buffer as one whose pointee could be const. Index the buffer instead, which says the same thing without a raw pointer at all. Signed-off-by: Niels Lohmann * Parenthesize the reserve arithmetic in the deep-nesting test clang-tidy's readability-math-missing-parentheses wants the multiplication spelled out in reserve(6 * depth + 1), and CI treats its warnings as errors. Signed-off-by: Niels Lohmann * Do not scan for a copyable run that cannot exist Under ensure_ascii, dump_escaped() calls find_ascii_copyable_run() at every character boundary. When the text is dense non-ASCII - CJK, where every byte is >= 0x80 - the scanner stops on its first byte and returns zero, so its SWAR block runs once per character and buys nothing, on top of the escaping that still has to happen afterwards. A run can only be non-empty when the first byte is one the scanner may copy, so test that single byte before calling it. Runs that do exist are found exactly as before, so the bulk-copy win is unchanged; only the calls that were always going to return zero are skipped. Output is unchanged: the dump digest over canada/citm/twitter, in compact, pretty and ensure_ascii form, matches develop byte for byte. dump(ensure_ascii=true) develop before after CJK text 3.54ms 4.25ms 3.36ms CJK, no ASCII at all 3.09ms 4.02ms 3.02ms Latin-1-ish text 4.39ms 3.04ms 2.93ms plain ASCII 3.92ms 0.80ms 0.79ms Signed-off-by: Niels Lohmann * Address review of the write-buffer helpers Three points from @gregmarr's review: put_chars() is gone. It was the only entry point taking a bare pointer and a count, and it existed only so put_string() and put_buffer() had something to delegate to. Its body now lives in put_string(), and put_buffer() is put_string(buffer, 0, length) - std::array already carries data() and size(), so it satisfies the same interface a string does. Nothing appends characters without a bound any more. dump_escaped()'s documentation block was duplicated. The dispatcher was inserted between the original comment and the function it described, and the comment was copied rather than split. The worker now has its own short comment saying why ensure_ascii is a template parameter. The local in dump_byte() is deliberate, and is now documented as such: writing through write_buffer[] is a char write, which may alias any object, so with write_buffer_pos updated in place the compiler must reload and store it around every digit. Measured on a dump of a 4 MiB binary value, 18.0 ms without the local against 7.4 ms with it. Output is unchanged: byte-identical dumps across 77 files in compact, pretty, ensure_ascii, pretty+ascii, indent 600 and tab-indent form. Signed-off-by: Niels Lohmann * Address review: drop unneeded backslash-escapes and duplicate scan loop '"' does not need escaping in a char literal, unlike in a string literal. find_ascii_copyable_run() also duplicated the byte-at-a-time search that already exists as the loop's own scalar tail; break into it instead of re-deriving the offset in a second, near-identical loop. Signed-off-by: Niels Lohmann * Move pretty_print, ensure_ascii and indent_step into the serializer None of these change over the life of a serializer, unlike current_indent and depth, which do change on every recursive call. They are now captured once in the constructor - matching indent_char and error_handler - instead of being threaded through dump(), dump_internal(), dump_iteratively(), dump_value() and dump_escaped() on every call. Signed-off-by: Niels Lohmann * Stop the serializer from holding onto std::localeconv()'s pointer loc was only ever read twice, immediately, to seed thousands_sep and decimal_point; nothing else in the class used it. A local in the constructor body serves the same purpose without keeping the pointer around for the serializer's lifetime. Signed-off-by: Niels Lohmann * Keep thousands_sep/decimal_point const via a small locale_chars struct const members can't be assigned in a constructor body, so seeding them from std::localeconv() meant either dropping const or holding onto the lconv* for longer than needed. A sub-object computes both from the pointer in its own constructor and is itself initialized in serializer's mem-initializer-list, so the two chars stay const, std::localeconv() is still called exactly once, and nothing outlives the constructor. Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann Co-authored-by: Claude Opus 4.8 --- include/nlohmann/detail/input/string_scan.hpp | 46 + include/nlohmann/detail/output/serializer.hpp | 967 +++++++++++++--- include/nlohmann/json.hpp | 14 +- single_include/nlohmann/json.hpp | 1028 ++++++++++++++--- tests/src/unit-convenience.cpp | 5 +- tests/src/unit-serialization.cpp | 229 ++++ 6 files changed, 2009 insertions(+), 280 deletions(-) diff --git a/include/nlohmann/detail/input/string_scan.hpp b/include/nlohmann/detail/input/string_scan.hpp index dc5b07a54..6af0e6c5d 100644 --- a/include/nlohmann/detail/input/string_scan.hpp +++ b/include/nlohmann/detail/input/string_scan.hpp @@ -93,6 +93,52 @@ inline std::size_t find_string_special(const unsigned char* data, std::size_t n) return n; } +// classify a byte as one the serializer must NOT copy verbatim when +// ensure_ascii is requested: the closing quote, an escape, a control character +// (< 0x20), DEL (0x7F), or any non-ASCII byte (>= 0x80). Everything else - +// printable ASCII except '"' and '\\' - is emitted unchanged. Note this differs +// from is_string_special() only in that 0x7F is also a stop (it is escaped as +// \u007f under ensure_ascii). +inline bool is_ascii_copyable(unsigned char c) noexcept +{ + return c >= 0x20u && c < 0x7Fu && c != '"' && c != '\\'; +} + +// return the index of the first byte in [data, data+n) that is NOT +// is_ascii_copyable(), or n if every byte can be copied verbatim; scans 8 bytes +// at a time. Used by the serializer's ensure_ascii fast path. +inline std::size_t find_ascii_copyable_run(const unsigned char* data, std::size_t n) noexcept +{ + constexpr std::uint64_t ones = 0x0101010101010101ull; + constexpr std::uint64_t high = 0x8080808080808080ull; + std::size_t i = 0; + for (; i + 8 <= n; i += 8) + { + std::uint64_t v = 0; + std::memcpy(&v, data + i, sizeof(v)); + const std::uint64_t q = v ^ 0x2222222222222222ull; // '"' (0x22) + const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; // '\\' (0x5C) + const std::uint64_t d = v ^ 0x7F7F7F7F7F7F7F7Full; // DEL (0x7F) + const std::uint64_t stop = ((q - ones) & ~q & high) // == '"' + | ((b - ones) & ~b & high) // == '\\' + | ((d - ones) & ~d & high) // == 0x7F + | ((v - 0x2020202020202020ull) & ~v & high) // < 0x20 + | (v & high); // >= 0x80 + if (stop != 0) + { + break; + } + } + for (; i < n; ++i) + { + if (!is_ascii_copyable(data[i])) + { + return i; + } + } + return n; +} + // Validate one UTF-8 sequence at the front of [data, data+avail). Returns its // length (2..4) only when the bytes form a *well-formed* sequence using exactly // the same ranges as scan_string()'s per-byte switch, so the bulk path accepts diff --git a/include/nlohmann/detail/output/serializer.hpp b/include/nlohmann/detail/output/serializer.hpp index 857fc2445..3dd9162df 100644 --- a/include/nlohmann/detail/output/serializer.hpp +++ b/include/nlohmann/detail/output/serializer.hpp @@ -9,20 +9,23 @@ #pragma once -#include // reverse, remove, fill, find, none_of +#include // reverse, remove, fill, find, none_of, min #include // array #include // localeconv, lconv #include // labs, isfinite, isnan, signbit #include // size_t, ptrdiff_t #include // uint8_t #include // snprintf +#include // memcpy, memset #include // numeric_limits #include // string, char_traits #include // is_same #include // move +#include // vector #include #include +#include #include #include #include @@ -61,16 +64,29 @@ class serializer /*! @param[in] s output stream to serialize to @param[in] ichar indentation character to use + @param[in] pretty_print_ whether the output shall be pretty-printed + @param[in] ensure_ascii_ If @a ensure_ascii_ is true, all non-ASCII + characters in the output are escaped with `\uXXXX` sequences, and the + result consists of ASCII characters only. + @param[in] indent_step_ the indent level @param[in] error_handler_ how to react on decoding errors + + None of @a pretty_print_, @a ensure_ascii_ and @a indent_step_ change over + the life of the serializer, so they are captured once here instead of + being threaded through every call to @ref dump, @ref dump_internal and + @ref dump_iteratively. */ serializer(output_adapter_t s, const char ichar, + const bool pretty_print_ = false, + const bool ensure_ascii_ = false, + const std::size_t indent_step_ = 0, error_handler_t error_handler_ = error_handler_t::strict) : o(std::move(s)) - , loc(std::localeconv()) - , thousands_sep(loc->thousands_sep == nullptr ? '\0' : std::char_traits::to_char_type(* (loc->thousands_sep))) - , decimal_point(loc->decimal_point == nullptr ? '\0' : std::char_traits::to_char_type(* (loc->decimal_point))) + , locale(std::localeconv()) , indent_char(ichar) - , indent_string(512, indent_char) + , pretty_print(pretty_print_) + , ensure_ascii(ensure_ascii_) + , indent_step(indent_step_) , error_handler(error_handler_) {} @@ -86,8 +102,8 @@ class serializer This function is called by the public member function dump and organizes the serialization internally. The indentation level is propagated as - additional parameter. In case of arrays and objects, the function is - called recursively. + additional parameter. Arrays and objects are serialized without recursion, + however deeply they are nested. - strings and object keys are escaped using `escape_string()` - integer numbers are converted implicitly via `operator<<` @@ -96,89 +112,109 @@ class serializer byte array @param[in] val value to serialize - @param[in] pretty_print whether the output shall be pretty-printed - @param[in] ensure_ascii If @a ensure_ascii is true, all non-ASCII characters - in the output are escaped with `\uXXXX` sequences, and the result consists - of ASCII characters only. - @param[in] indent_step the indent level @param[in] current_indent the current indent level (only used internally) */ void dump(const BasicJsonType& val, - const bool pretty_print, - const bool ensure_ascii, - const unsigned int indent_step, - const unsigned int current_indent = 0) + const std::size_t current_indent = 0) + { + dump_internal(val, current_indent); + flush(); + } + + JSON_PRIVATE_UNLESS_TESTED: + /*! + @brief worker for @ref dump + + Identical in behavior to the historical @ref dump, but writes into the + serializer's internal @ref write_buffer instead of issuing a virtual call + per token. The public @ref dump wraps this and flushes the buffer once the + top-level value has been serialized. + + Serializing a container descends into its elements, so a value nested deeply + enough used to exhaust the call stack and terminate the process with no + exception to catch. The descent is bounded here: once @ref dump_depth_limit + levels have been entered, @ref dump_iteratively writes out what is left + without the call stack. A value nested less deeply than that - all but a + vanishing minority - is written by exactly the code that always wrote it. + + @sa https://github.com/nlohmann/json/issues/5387 + */ + void dump_internal(const BasicJsonType& val, + const std::size_t current_indent = 0, + const std::size_t depth = 0) { switch (val.m_data.m_type) { case value_t::object: { + if (JSON_HEDLEY_UNLIKELY(depth >= dump_depth_limit())) + { + dump_iteratively(val, current_indent); + return; + } + if (val.m_data.m_value.object->empty()) { - o->write_characters("{}", 2); + put_literal("{}"); return; } if (pretty_print) { - o->write_characters("{\n", 2); + put_literal("{\n"); // variable to hold indentation for recursive calls - const auto new_indent = current_indent + indent_step; - if (JSON_HEDLEY_UNLIKELY(indent_string.size() < new_indent)) - { - indent_string.resize(indent_string.size() * 2, ' '); - } + const auto new_indent = next_indent(current_indent, indent_step); // first n-1 elements auto i = val.m_data.m_value.object->cbegin(); for (std::size_t cnt = 0; cnt < val.m_data.m_value.object->size() - 1; ++cnt, ++i) { - o->write_characters(indent_string.c_str(), new_indent); - o->write_character('\"'); - dump_escaped(i->first, ensure_ascii); - o->write_characters("\": ", 3); - dump(i->second, true, ensure_ascii, indent_step, new_indent); - o->write_characters(",\n", 2); + put_indent(new_indent); + put_char('"'); + dump_escaped(i->first); + put_literal("\": "); + dump_internal(i->second, new_indent, depth + 1); + put_literal(",\n"); } // last element JSON_ASSERT(i != val.m_data.m_value.object->cend()); JSON_ASSERT(std::next(i) == val.m_data.m_value.object->cend()); - o->write_characters(indent_string.c_str(), new_indent); - o->write_character('\"'); - dump_escaped(i->first, ensure_ascii); - o->write_characters("\": ", 3); - dump(i->second, true, ensure_ascii, indent_step, new_indent); + put_indent(new_indent); + put_char('"'); + dump_escaped(i->first); + put_literal("\": "); + dump_internal(i->second, new_indent, depth + 1); - o->write_character('\n'); - o->write_characters(indent_string.c_str(), current_indent); - o->write_character('}'); + put_char('\n'); + put_indent(current_indent); + put_char('}'); } else { - o->write_character('{'); + put_char('{'); // first n-1 elements auto i = val.m_data.m_value.object->cbegin(); for (std::size_t cnt = 0; cnt < val.m_data.m_value.object->size() - 1; ++cnt, ++i) { - o->write_character('\"'); - dump_escaped(i->first, ensure_ascii); - o->write_characters("\":", 2); - dump(i->second, false, ensure_ascii, indent_step, current_indent); - o->write_character(','); + put_char('"'); + dump_escaped(i->first); + put_literal("\":"); + dump_internal(i->second, current_indent, depth + 1); + put_char(','); } // last element JSON_ASSERT(i != val.m_data.m_value.object->cend()); JSON_ASSERT(std::next(i) == val.m_data.m_value.object->cend()); - o->write_character('\"'); - dump_escaped(i->first, ensure_ascii); - o->write_characters("\":", 2); - dump(i->second, false, ensure_ascii, indent_step, current_indent); + put_char('"'); + dump_escaped(i->first); + put_literal("\":"); + dump_internal(i->second, current_indent, depth + 1); - o->write_character('}'); + put_char('}'); } return; @@ -186,58 +222,60 @@ class serializer case value_t::array: { + if (JSON_HEDLEY_UNLIKELY(depth >= dump_depth_limit())) + { + dump_iteratively(val, current_indent); + return; + } + if (val.m_data.m_value.array->empty()) { - o->write_characters("[]", 2); + put_literal("[]"); return; } if (pretty_print) { - o->write_characters("[\n", 2); + put_literal("[\n"); // variable to hold indentation for recursive calls - const auto new_indent = current_indent + indent_step; - if (JSON_HEDLEY_UNLIKELY(indent_string.size() < new_indent)) - { - indent_string.resize(indent_string.size() * 2, ' '); - } + const auto new_indent = next_indent(current_indent, indent_step); // first n-1 elements for (auto i = val.m_data.m_value.array->cbegin(); i != val.m_data.m_value.array->cend() - 1; ++i) { - o->write_characters(indent_string.c_str(), new_indent); - dump(*i, true, ensure_ascii, indent_step, new_indent); - o->write_characters(",\n", 2); + put_indent(new_indent); + dump_internal(*i, new_indent, depth + 1); + put_literal(",\n"); } // last element JSON_ASSERT(!val.m_data.m_value.array->empty()); - o->write_characters(indent_string.c_str(), new_indent); - dump(val.m_data.m_value.array->back(), true, ensure_ascii, indent_step, new_indent); + put_indent(new_indent); + dump_internal(val.m_data.m_value.array->back(), new_indent, depth + 1); - o->write_character('\n'); - o->write_characters(indent_string.c_str(), current_indent); - o->write_character(']'); + put_char('\n'); + put_indent(current_indent); + put_char(']'); } else { - o->write_character('['); + put_char('['); // first n-1 elements for (auto i = val.m_data.m_value.array->cbegin(); i != val.m_data.m_value.array->cend() - 1; ++i) { - dump(*i, false, ensure_ascii, indent_step, current_indent); - o->write_character(','); + dump_internal(*i, current_indent, depth + 1); + put_char(','); } // last element JSON_ASSERT(!val.m_data.m_value.array->empty()); - dump(val.m_data.m_value.array->back(), false, ensure_ascii, indent_step, current_indent); + dump_internal(val.m_data.m_value.array->back(), current_indent, depth + 1); - o->write_character(']'); + put_char(']'); } return; @@ -245,9 +283,9 @@ class serializer case value_t::string: { - o->write_character('\"'); - dump_escaped(*val.m_data.m_value.string, ensure_ascii); - o->write_character('\"'); + put_char('"'); + dump_escaped(*val.m_data.m_value.string); + put_char('"'); return; } @@ -255,70 +293,66 @@ class serializer { if (pretty_print) { - o->write_characters("{\n", 2); + put_literal("{\n"); // variable to hold indentation for recursive calls - const auto new_indent = current_indent + indent_step; - if (JSON_HEDLEY_UNLIKELY(indent_string.size() < new_indent)) - { - indent_string.resize(indent_string.size() * 2, ' '); - } + const auto new_indent = next_indent(current_indent, indent_step); - o->write_characters(indent_string.c_str(), new_indent); + put_indent(new_indent); - o->write_characters("\"bytes\": [", 10); + put_literal("\"bytes\": ["); if (!val.m_data.m_value.binary->empty()) { for (auto i = val.m_data.m_value.binary->cbegin(); i != val.m_data.m_value.binary->cend() - 1; ++i) { - dump_integer(*i); - o->write_characters(", ", 2); + dump_byte(*i); + put_literal(", "); } - dump_integer(val.m_data.m_value.binary->back()); + dump_byte(val.m_data.m_value.binary->back()); } - o->write_characters("],\n", 3); - o->write_characters(indent_string.c_str(), new_indent); + put_literal("],\n"); + put_indent(new_indent); - o->write_characters("\"subtype\": ", 11); + put_literal("\"subtype\": "); if (val.m_data.m_value.binary->has_subtype()) { dump_integer(val.m_data.m_value.binary->subtype()); } else { - o->write_characters("null", 4); + put_literal("null"); } - o->write_character('\n'); - o->write_characters(indent_string.c_str(), current_indent); - o->write_character('}'); + put_char('\n'); + put_indent(current_indent); + put_char('}'); } else { - o->write_characters("{\"bytes\":[", 10); + put_literal("{\"bytes\":["); if (!val.m_data.m_value.binary->empty()) { for (auto i = val.m_data.m_value.binary->cbegin(); i != val.m_data.m_value.binary->cend() - 1; ++i) { - dump_integer(*i); - o->write_character(','); + dump_byte(*i); + put_char(','); } - dump_integer(val.m_data.m_value.binary->back()); + dump_byte(val.m_data.m_value.binary->back()); } - o->write_characters("],\"subtype\":", 12); + put_literal("],\"subtype\":"); if (val.m_data.m_value.binary->has_subtype()) { dump_integer(val.m_data.m_value.binary->subtype()); - o->write_character('}'); + put_char('}'); } else { - o->write_characters("null}", 5); + put_literal("null}"); } } return; @@ -328,11 +362,11 @@ class serializer { if (val.m_data.m_value.boolean) { - o->write_characters("true", 4); + put_literal("true"); } else { - o->write_characters("false", 5); + put_literal("false"); } return; } @@ -357,13 +391,13 @@ class serializer case value_t::discarded: { - o->write_characters("", 11); + put_literal(""); return; } case value_t::null: { - o->write_characters("null", 4); + put_literal("null"); return; } @@ -372,6 +406,367 @@ class serializer } } + private: + /// the number of levels @ref dump_internal descends into before it hands + /// over to @ref dump_iteratively + static constexpr std::size_t dump_depth_limit() + { + return 128; + } + + /*! + @brief write out @a val and everything below it without the call stack + + Emits the same bytes as @ref dump_internal, keeping the containers it has + entered on an explicit stack instead of descending into them. Only reached + for values nested deeper than @ref dump_depth_limit, which is why it is not + written for speed: walking every value this way measured up to 20% slower on + object-heavy documents than letting the compiler drive the descent. + */ + void dump_iteratively(const BasicJsonType& val, + const std::size_t current_indent = 0) + { + // Scalars, empty containers and binary values are written by dump_value + // alone, so nothing is allocated for them: only a container with + // elements is ever pushed. + std::vector stack; + + dump_value(val, current_indent, stack); + + while (!stack.empty()) + { + dump_frame& frame = stack.back(); + + if (frame.value->m_data.m_type == value_t::object) + { + const auto* object = frame.value->m_data.m_value.object; + + if (frame.object_it == object->cend()) + { + if (pretty_print) + { + put_char('\n'); + put_indent(frame.current_indent); + } + + put_char('}'); + stack.pop_back(); + continue; + } + + // the separator goes in front of every element but the first, + // which puts exactly one between each pair and none at the end + if (frame.object_it != object->cbegin()) + { + if (pretty_print) + { + put_literal(",\n"); + } + else + { + put_char(','); + } + } + + if (pretty_print) + { + put_indent(frame.child_indent); + } + + put_char('"'); + dump_escaped(frame.object_it->first); + + if (pretty_print) + { + put_literal("\": "); + } + else + { + put_literal("\":"); + } + + const BasicJsonType& element = frame.object_it->second; + ++frame.object_it; + + // read everything needed from the frame before this: entering a + // container pushes another one and can move them all + const std::size_t element_indent = frame.child_indent; + dump_value(element, element_indent, stack); + } + else + { + const auto* array = frame.value->m_data.m_value.array; + + if (frame.array_it == array->cend()) + { + if (pretty_print) + { + put_char('\n'); + put_indent(frame.current_indent); + } + + put_char(']'); + stack.pop_back(); + continue; + } + + if (frame.array_it != array->cbegin()) + { + if (pretty_print) + { + put_literal(",\n"); + } + else + { + put_char(','); + } + } + + if (pretty_print) + { + put_indent(frame.child_indent); + } + + const BasicJsonType& element = *frame.array_it; + ++frame.array_it; + + // see above + const std::size_t element_indent = frame.child_indent; + dump_value(element, element_indent, stack); + } + } + } + + private: + /// @brief a container that has been opened but not closed yet + struct dump_frame + { + dump_frame(const BasicJsonType* value_, const std::size_t current_indent_, + const std::size_t child_indent_) noexcept + : value(value_) + , current_indent(current_indent_) + , child_indent(child_indent_) + {} + + /// the object or array being serialized + const BasicJsonType* value; + /// the element to serialize next; which of the two is live follows from + /// the type of @a value. They are kept side by side rather than in a + /// union, which would need its special members written out by hand, see + /// detail/iterators/internal_iterator.hpp + typename BasicJsonType::object_t::const_iterator object_it{}; + typename BasicJsonType::array_t::const_iterator array_it{}; + /// the indentation of the container itself, used by its closing bracket + std::size_t current_indent; + /// the indentation of the container's elements + std::size_t child_indent; + }; + + /*! + @brief serialize the value @a val, but not the elements of a container + + An object or array with elements is opened and pushed onto @a stack for + @ref dump_internal to walk; everything else - including a binary value, + which looks like an object but has no elements to descend into - is written + out here in full. + */ + void dump_value(const BasicJsonType& val, + const std::size_t current_indent, + std::vector& stack) + { + switch (val.m_data.m_type) + { + case value_t::object: + { + if (val.m_data.m_value.object->empty()) + { + put_literal("{}"); + return; + } + + std::size_t child_indent = current_indent; + + if (pretty_print) + { + put_literal("{\n"); + child_indent = next_indent(current_indent, indent_step); + } + else + { + put_char('{'); + } + + stack.emplace_back(&val, current_indent, child_indent); + stack.back().object_it = val.m_data.m_value.object->cbegin(); + return; + } + + case value_t::array: + { + if (val.m_data.m_value.array->empty()) + { + put_literal("[]"); + return; + } + + std::size_t child_indent = current_indent; + + if (pretty_print) + { + put_literal("[\n"); + child_indent = next_indent(current_indent, indent_step); + } + else + { + put_char('['); + } + + stack.emplace_back(&val, current_indent, child_indent); + stack.back().array_it = val.m_data.m_value.array->cbegin(); + return; + } + + case value_t::string: + { + put_char('"'); + dump_escaped(*val.m_data.m_value.string); + put_char('"'); + return; + } + + case value_t::binary: + { + if (pretty_print) + { + put_literal("{\n"); + + // variable to hold indentation for the bytes + const auto new_indent = next_indent(current_indent, indent_step); + + put_indent(new_indent); + + put_literal("\"bytes\": ["); + + if (!val.m_data.m_value.binary->empty()) + { + for (auto i = val.m_data.m_value.binary->cbegin(); + i != val.m_data.m_value.binary->cend() - 1; ++i) + { + dump_byte(*i); + put_literal(", "); + } + dump_byte(val.m_data.m_value.binary->back()); + } + + put_literal("],\n"); + put_indent(new_indent); + + put_literal("\"subtype\": "); + if (val.m_data.m_value.binary->has_subtype()) + { + dump_integer(val.m_data.m_value.binary->subtype()); + } + else + { + put_literal("null"); + } + put_char('\n'); + put_indent(current_indent); + put_char('}'); + } + else + { + put_literal("{\"bytes\":["); + + if (!val.m_data.m_value.binary->empty()) + { + for (auto i = val.m_data.m_value.binary->cbegin(); + i != val.m_data.m_value.binary->cend() - 1; ++i) + { + dump_byte(*i); + put_char(','); + } + dump_byte(val.m_data.m_value.binary->back()); + } + + put_literal("],\"subtype\":"); + if (val.m_data.m_value.binary->has_subtype()) + { + dump_integer(val.m_data.m_value.binary->subtype()); + put_char('}'); + } + else + { + put_literal("null}"); + } + } + return; + } + + case value_t::boolean: + { + if (val.m_data.m_value.boolean) + { + put_literal("true"); + } + else + { + put_literal("false"); + } + return; + } + + case value_t::number_integer: + { + dump_integer(val.m_data.m_value.number_integer); + return; + } + + case value_t::number_unsigned: + { + dump_integer(val.m_data.m_value.number_unsigned); + return; + } + + case value_t::number_float: + { + dump_float(val.m_data.m_value.number_float); + return; + } + + case value_t::discarded: + { + put_literal(""); + return; + } + + case value_t::null: + { + put_literal("null"); + return; + } + + default: // LCOV_EXCL_LINE + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE + } + } + + + /*! + @brief the indentation level to use for the children of the current value + + A very large @a indent_step can wrap the unsigned accumulation on deep + nesting, which would silently truncate the indentation. Far harder to reach + now that the accumulator is a std::size_t, but still reachable where that is + 32 bits wide. + */ + static std::size_t next_indent(const std::size_t current_indent, const std::size_t indent_step) + { + const std::size_t new_indent = current_indent + indent_step; + JSON_ASSERT(new_indent >= current_indent); + return new_indent; + } + JSON_PRIVATE_UNLESS_TESTED: /*! @brief dump escaped string @@ -382,12 +777,32 @@ class serializer representation. The escaped string is written to output stream @a o. @param[in] s the string to escape - @param[in] ensure_ascii whether to escape non-ASCII characters with - \uXXXX sequences @complexity Linear in the length of string @a s. */ - void dump_escaped(const string_t& s, const bool ensure_ascii) + void dump_escaped(const string_t& s) + { + // dispatch once here rather than test the flag inside the loop: it does + // not change while a string is written, and folding it lets each of the + // two scanners be inlined into a loop of its own + if (ensure_ascii) + { + dump_escaped_impl(s); + } + else + { + dump_escaped_impl(s); + } + } + + /*! + @brief worker for @ref dump_escaped + + @a ensure_ascii is a template parameter here so that the branch on it is + resolved once, outside the loop; see @ref dump_escaped. + */ + template + void dump_escaped_impl(const string_t& s) { std::uint32_t codepoint{}; std::uint8_t state = UTF8_ACCEPT; @@ -399,6 +814,56 @@ class serializer for (std::size_t i = 0; i < s.size(); ++i) { + // Fast path: at a character boundary (state == UTF8_ACCEPT), + // bulk-copy the longest run of bytes that need no escaping using a + // SWAR scanner shared with the lexer's contiguous path. The scanner + // stops exactly at the first byte dump_escaped would handle + // individually, so that byte is left to the byte-at-a-time path + // below, keeping escaping output and error diagnostics unchanged. + // + // - EnsureAscii == false: string_bulk_run() copies ordinary bytes + // and complete well-formed UTF-8, stopping at a quote, backslash, + // control character (< 0x20), or ill-formed/truncated sequence. + // - EnsureAscii == true: only printable ASCII may be copied + // verbatim; find_ascii_copyable_run() additionally stops at 0x7F + // and every non-ASCII byte (>= 0x80), which must be \u-escaped. + if (state == UTF8_ACCEPT) + { + const auto* const data = reinterpret_cast(s.data()); + // A run can only be non-empty when the very first byte is one + // the scanner may copy, so test that single byte before paying + // for the scan. Without it, text whose characters all have to be + // escaped - CJK under ensure_ascii, where every byte is >= 0x80 - + // runs the scanner once per character only to be told zero. + std::size_t run = 0; + if (!EnsureAscii) + { + run = string_bulk_run(data + i, s.size() - i); + } + else if (is_ascii_copyable(data[i])) + { + run = find_ascii_copyable_run(data + i, s.size() - i); + } + if (run != 0) + { + // emit any bytes still pending in string_buffer first to + // preserve output order, then write the run directly + if (bytes != 0) + { + put_buffer(string_buffer, bytes); + bytes = 0; + } + put_string(s, i, i + run); + bytes_after_last_accept = 0; + undumped_chars = 0; + i += run; + if (i >= s.size()) + { + break; + } + } + } + const auto byte = static_cast(s[i]); switch (decode(state, codepoint, byte)) @@ -445,7 +910,7 @@ class serializer case 0x22: // quotation mark { string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = '\"'; + string_buffer[bytes++] = '"'; break; } @@ -459,8 +924,8 @@ class serializer default: { // escape control characters (0x00..0x1F) or, if - // ensure_ascii parameter is used, non-ASCII characters - if ((codepoint <= 0x1F) || (ensure_ascii && (codepoint >= 0x7F))) + // EnsureAscii parameter is used, non-ASCII characters + if ((codepoint <= 0x1F) || (EnsureAscii && (codepoint >= 0x7F))) { if (codepoint <= 0xFFFF) { @@ -487,7 +952,7 @@ class serializer // written ("\uxxxx\uxxxx\0") for one code point if (string_buffer.size() - bytes < 13) { - o->write_characters(string_buffer.data(), bytes); + put_buffer(string_buffer, bytes); bytes = 0; } @@ -525,7 +990,7 @@ class serializer if (error_handler == error_handler_t::replace) { // add a replacement character - if (ensure_ascii) + if (EnsureAscii) { string_buffer[bytes++] = '\\'; string_buffer[bytes++] = 'u'; @@ -546,7 +1011,7 @@ class serializer // written ("\uxxxx\uxxxx\0") for one code point if (string_buffer.size() - bytes < 13) { - o->write_characters(string_buffer.data(), bytes); + put_buffer(string_buffer, bytes); bytes = 0; } @@ -568,7 +1033,7 @@ class serializer default: // decode found yet incomplete multibyte code point { - if (!ensure_ascii) + if (!EnsureAscii) { // code point will not be escaped - copy byte to buffer string_buffer[bytes++] = s[i]; @@ -585,7 +1050,7 @@ class serializer // write buffer if (bytes > 0) { - o->write_characters(string_buffer.data(), bytes); + put_buffer(string_buffer, bytes); } } else @@ -601,22 +1066,22 @@ class serializer case error_handler_t::ignore: { // write all accepted bytes - o->write_characters(string_buffer.data(), bytes_after_last_accept); + put_buffer(string_buffer, bytes_after_last_accept); break; } case error_handler_t::replace: { // write all accepted bytes - o->write_characters(string_buffer.data(), bytes_after_last_accept); + put_buffer(string_buffer, bytes_after_last_accept); // add a replacement character - if (ensure_ascii) + if (EnsureAscii) { - o->write_characters("\\ufffd", 6); + put_literal("\\ufffd"); } else { - o->write_characters("\xEF\xBF\xBD", 3); + put_literal("\xEF\xBF\xBD"); } break; } @@ -627,6 +1092,160 @@ class serializer } } + private: + /*! + @brief append a single character to the write buffer + + Structural characters ('{', '"', ',', ...) previously went straight to the + output adapter, one virtual call each. Buffering them and flushing in bulk + turns those many indirect calls into a single memcpy plus an occasional + flush, which dominates the cost of serializing object/array-heavy values. + */ + void put_char(char c) + { + if (JSON_HEDLEY_UNLIKELY(write_buffer_pos == write_buffer.size())) + { + flush(); + } + write_buffer[write_buffer_pos++] = c; + } + + /*! + @brief append @a indent indentation characters to the write buffer + + Writes the indentation straight into the buffer instead of copying it out of + a pre-grown indentation string, so no auxiliary string has to be sized, + resized, or kept in sync with the deepest nesting level reached. + + An indentation wider than the buffer is emitted by filling the buffer with + the indentation character once and flushing that same content repeatedly: + flushing does not disturb what the buffer holds, so re-filling it between + flushes would be redundant work. + */ + void put_indent(std::size_t indent) + { + // closing braces at the outermost level ask for no indentation at all + if (indent == 0) + { + return; + } + + const std::size_t capacity = write_buffer.size(); + + // fill whatever room is left in the buffer; this is the whole job + // whenever the indentation is narrower than the buffer, which is the + // case for every sane indent_step + const std::size_t head = (std::min)(indent, capacity - write_buffer_pos); + std::memset(write_buffer.data() + write_buffer_pos, indent_char, head); + write_buffer_pos += head; + indent -= head; + + if (JSON_HEDLEY_LIKELY(indent == 0)) + { + return; + } + + // the buffer is full and the remainder spans whole buffer-fulls: flush + // what is pending, then fill the buffer with the indentation character + // exactly once and hand the same bytes to the adapter as often as needed + flush(); + std::memset(write_buffer.data(), indent_char, capacity); + + while (indent >= capacity) + { + write_buffer_pos = capacity; + flush(); + indent -= capacity; + } + + // the buffer still holds indentation characters throughout, so the tail + // only has to be claimed, not written again + write_buffer_pos = indent; + } + + /*! + @brief append a string literal to the write buffer + + The length comes from the array bound rather than a hand-written count, so + it cannot drift out of sync with the literal. A literal always fits into the + buffer (checked at compile time), so unlike @ref put_string this needs no + write-through path for oversized runs. + */ + template + void put_literal(const char (&s)[N]) // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + { + static_assert(N >= 2, "put_literal expects a non-empty string literal"); + // the array bound counts the terminating NUL, which is not written + constexpr std::size_t length = N - 1; + static_assert(length < write_buffer_size, "string literal must fit into the write buffer"); + + if (JSON_HEDLEY_UNLIKELY(write_buffer_pos + length > write_buffer.size())) + { + flush(); + } + std::memcpy(write_buffer.data() + write_buffer_pos, s, length); + write_buffer_pos += length; + } + + /*! + @brief append the characters of @a str in [@a start, @a end) + + The only way to append a run of characters: @a str carries its own bound, + so the range can be checked against it, which a bare pointer plus a count + could not do. Runs that do not fit the buffer are written straight through + the output adapter (after flushing what is pending), so large string and + number payloads are not copied an extra time. + */ + template + void put_string(const StringType& str, std::size_t start, std::size_t end) + { + JSON_ASSERT(start <= end); + JSON_ASSERT(end <= str.size()); + + const char* const s = str.data() + start; + const std::size_t length = end - start; + + if (JSON_HEDLEY_UNLIKELY(length >= write_buffer.size())) + { + flush(); + o->write_characters(s, length); + return; + } + if (JSON_HEDLEY_UNLIKELY(write_buffer_pos + length > write_buffer.size())) + { + flush(); + } + std::memcpy(write_buffer.data() + write_buffer_pos, s, length); + write_buffer_pos += length; + } + + /*! + @brief append the first @a length characters of a fixed-size buffer + */ + template + void put_buffer(const std::array& buffer, std::size_t length) + { + put_string(buffer, 0, length); + } + + JSON_PRIVATE_UNLESS_TESTED: + /*! + @brief flush the write buffer to the output adapter + + Writing zero characters is a well-defined no-op for every output adapter, so + the buffered length is passed through unconditionally (no empty-guard branch + to leave uncovered). + + @note dump_escaped() and dump_integer()/dump_float() write into the internal + write buffer; callers that invoke them directly (rather than through the + public dump()) must call flush() before inspecting the output. + */ + void flush() + { + o->write_characters(write_buffer.data(), write_buffer_pos); + write_buffer_pos = 0; + } + private: /*! @brief count digits @@ -715,6 +1334,62 @@ class serializer return false; } + /*! + @brief write the decimal representation of the byte @a value + + A binary value's bytes are always in [0, 255], so writing one needs neither + the digit counting nor the 64-bit arithmetic that @ref dump_integer does for + an arbitrary number, and the three digits it takes at most are written + straight into the write buffer. + + Any byte type that is not a plain unsigned byte is left to @ref dump_integer, + whose representation of it may differ. + */ + template + void dump_byte(const ByteType value) + { + dump_byte(value, std::integral_constant < bool, + std::is_unsigned::value && sizeof(ByteType) == 1 + && !std::is_same::value > {}); + } + + template + void dump_byte(const ByteType value, std::false_type /*is_plain_byte*/) + { + dump_integer(value); + } + + template + void dump_byte(const ByteType value, std::true_type /*is_plain_byte*/) + { + if (JSON_HEDLEY_UNLIKELY(write_buffer_pos + 3 > write_buffer.size())) + { + flush(); + } + + const auto byte = static_cast(value); + // Accumulate the offset in a local and store it back once. Writing + // through write_buffer[] is a char write, which may alias any object, + // so with the member updated in place the compiler has to reload and + // store it around every digit - measured 2.4x slower on a dump of a + // multi-megabyte binary value. + std::size_t pos = write_buffer_pos; + + if (byte >= 100) + { + write_buffer[pos++] = static_cast('0' + (byte / 100)); + write_buffer[pos++] = static_cast('0' + ((byte / 10) % 10)); + } + else if (byte >= 10) + { + write_buffer[pos++] = static_cast('0' + (byte / 10)); + } + + write_buffer[pos++] = static_cast('0' + (byte % 10)); + + write_buffer_pos = pos; + } + /*! @brief dump an integer @@ -751,7 +1426,7 @@ class serializer // special case for "0" if (x == 0) { - o->write_character('0'); + put_char('0'); return; } @@ -804,7 +1479,7 @@ class serializer *(--buffer_ptr) = static_cast('0' + abs_value); } - o->write_characters(number_buffer.data(), n_chars); + put_buffer(number_buffer, n_chars); } /*! @@ -820,7 +1495,7 @@ class serializer // NaN / inf if (!std::isfinite(x)) { - o->write_characters("null", 4); + put_literal("null"); return; } @@ -841,7 +1516,7 @@ class serializer auto* begin = number_buffer.data(); auto* end = ::nlohmann::detail::to_chars(begin, begin + number_buffer.size(), x); - o->write_characters(begin, static_cast(end - begin)); + put_buffer(number_buffer, static_cast(end - begin)); } JSON_HEDLEY_NON_NULL(1) @@ -872,27 +1547,27 @@ class serializer JSON_ASSERT(static_cast(len) < number_buffer.size()); // erase thousands separators - if (thousands_sep != '\0') + if (locale.thousands_sep != '\0') { // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::remove returns an iterator, see https://github.com/nlohmann/json/issues/3081 - const auto end = std::remove(number_buffer.begin(), number_buffer.begin() + len, thousands_sep); + const auto end = std::remove(number_buffer.begin(), number_buffer.begin() + len, locale.thousands_sep); std::fill(end, number_buffer.end(), '\0'); JSON_ASSERT((end - number_buffer.begin()) <= len); len = (end - number_buffer.begin()); } // convert decimal point to '.' - if (decimal_point != '\0' && decimal_point != '.') + if (locale.decimal_point != '\0' && locale.decimal_point != '.') { // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::find returns an iterator, see https://github.com/nlohmann/json/issues/3081 - const auto dec_pos = std::find(number_buffer.begin(), number_buffer.end(), decimal_point); + const auto dec_pos = std::find(number_buffer.begin(), number_buffer.end(), locale.decimal_point); if (dec_pos != number_buffer.end()) { *dec_pos = '.'; } } - o->write_characters(number_buffer.data(), static_cast(len)); + put_buffer(number_buffer, static_cast(len)); // determine if we need to append ".0" const bool value_is_int_like = @@ -904,7 +1579,7 @@ class serializer if (value_is_int_like) { - o->write_characters(".0", 2); + put_literal(".0"); } } @@ -991,29 +1666,53 @@ class serializer } private: + /// the locale's thousand separator and decimal point characters + struct locale_chars + { + explicit locale_chars(const std::lconv* loc) noexcept + : thousands_sep(loc->thousands_sep == nullptr ? '\0' : std::char_traits::to_char_type(* (loc->thousands_sep))) + , decimal_point(loc->decimal_point == nullptr ? '\0' : std::char_traits::to_char_type(* (loc->decimal_point))) + {} + + const char thousands_sep; + const char decimal_point; + }; + /// the output of the serializer output_adapter_t o = nullptr; /// a (hopefully) large enough character buffer std::array number_buffer{{}}; - /// the locale - const std::lconv* loc = nullptr; - /// the locale's thousand separator character - const char thousands_sep = '\0'; - /// the locale's decimal point character - const char decimal_point = '\0'; + /// computed once from std::localeconv() at construction; @ref + /// locale_chars keeps std::localeconv()'s pointer from having to be held + /// past the constructor, while still letting these stay const + const locale_chars locale; /// string buffer std::array string_buffer{{}}; /// the indentation character const char indent_char; - /// the indentation string - string_t indent_string; + + /// whether to pretty-print the output + const bool pretty_print; + + /// whether to escape non-ASCII characters with \uXXXX sequences + const bool ensure_ascii; + + /// the indent level + const std::size_t indent_step; /// error_handler how to react on decoding errors const error_handler_t error_handler; + + /// buffer collecting output before it is flushed to the output adapter, so + /// that the many small structural writes become few bulk writes + static constexpr std::size_t write_buffer_size = 1024; + std::array write_buffer{{}}; + /// number of valid bytes currently held in @ref write_buffer + std::size_t write_buffer_pos = 0; }; } // namespace detail diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 9bbd98f15..7243c3c44 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -1343,15 +1343,18 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const error_handler_t error_handler = error_handler_t::strict) const { string_t result; - serializer s(detail::output_adapter(result), indent_char, error_handler); if (indent >= 0) { - s.dump(*this, true, ensure_ascii, static_cast(indent)); + serializer s(detail::output_adapter(result), indent_char, + true, ensure_ascii, static_cast(indent), error_handler); + s.dump(*this); } else { - s.dump(*this, false, ensure_ascii, 0); + serializer s(detail::output_adapter(result), indent_char, + false, ensure_ascii, 0, error_handler); + s.dump(*this); } return result; @@ -4080,8 +4083,9 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec o.width(0); // do the actual serialization - serializer s(detail::output_adapter(o), o.fill()); - s.dump(j, pretty_print, false, static_cast(indentation)); + serializer s(detail::output_adapter(o), o.fill(), + pretty_print, false, static_cast(indentation)); + s.dump(j); return o; } diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 34db800b7..8b1f4506e 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -8327,6 +8327,52 @@ inline std::size_t find_string_special(const unsigned char* data, std::size_t n) return n; } +// classify a byte as one the serializer must NOT copy verbatim when +// ensure_ascii is requested: the closing quote, an escape, a control character +// (< 0x20), DEL (0x7F), or any non-ASCII byte (>= 0x80). Everything else - +// printable ASCII except '"' and '\\' - is emitted unchanged. Note this differs +// from is_string_special() only in that 0x7F is also a stop (it is escaped as +// \u007f under ensure_ascii). +inline bool is_ascii_copyable(unsigned char c) noexcept +{ + return c >= 0x20u && c < 0x7Fu && c != '"' && c != '\\'; +} + +// return the index of the first byte in [data, data+n) that is NOT +// is_ascii_copyable(), or n if every byte can be copied verbatim; scans 8 bytes +// at a time. Used by the serializer's ensure_ascii fast path. +inline std::size_t find_ascii_copyable_run(const unsigned char* data, std::size_t n) noexcept +{ + constexpr std::uint64_t ones = 0x0101010101010101ull; + constexpr std::uint64_t high = 0x8080808080808080ull; + std::size_t i = 0; + for (; i + 8 <= n; i += 8) + { + std::uint64_t v = 0; + std::memcpy(&v, data + i, sizeof(v)); + const std::uint64_t q = v ^ 0x2222222222222222ull; // '"' (0x22) + const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; // '\\' (0x5C) + const std::uint64_t d = v ^ 0x7F7F7F7F7F7F7F7Full; // DEL (0x7F) + const std::uint64_t stop = ((q - ones) & ~q & high) // == '"' + | ((b - ones) & ~b & high) // == '\\' + | ((d - ones) & ~d & high) // == 0x7F + | ((v - 0x2020202020202020ull) & ~v & high) // < 0x20 + | (v & high); // >= 0x80 + if (stop != 0) + { + break; + } + } + for (; i < n; ++i) + { + if (!is_ascii_copyable(data[i])) + { + return i; + } + } + return n; +} + // Validate one UTF-8 sequence at the front of [data, data+avail). Returns its // length (2..4) only when the bytes form a *well-formed* sequence using exactly // the same ranges as scan_string()'s per-byte switch, so the bulk path accepts @@ -20183,17 +20229,19 @@ NLOHMANN_JSON_NAMESPACE_END -#include // reverse, remove, fill, find, none_of +#include // reverse, remove, fill, find, none_of, min #include // array #include // localeconv, lconv #include // labs, isfinite, isnan, signbit #include // size_t, ptrdiff_t #include // uint8_t #include // snprintf +#include // memcpy, memset #include // numeric_limits #include // string, char_traits #include // is_same #include // move +#include // vector // #include // __ _____ _____ _____ @@ -21318,6 +21366,8 @@ NLOHMANN_JSON_NAMESPACE_END // #include +// #include + // #include // #include @@ -21362,16 +21412,29 @@ class serializer /*! @param[in] s output stream to serialize to @param[in] ichar indentation character to use + @param[in] pretty_print_ whether the output shall be pretty-printed + @param[in] ensure_ascii_ If @a ensure_ascii_ is true, all non-ASCII + characters in the output are escaped with `\uXXXX` sequences, and the + result consists of ASCII characters only. + @param[in] indent_step_ the indent level @param[in] error_handler_ how to react on decoding errors + + None of @a pretty_print_, @a ensure_ascii_ and @a indent_step_ change over + the life of the serializer, so they are captured once here instead of + being threaded through every call to @ref dump, @ref dump_internal and + @ref dump_iteratively. */ serializer(output_adapter_t s, const char ichar, + const bool pretty_print_ = false, + const bool ensure_ascii_ = false, + const std::size_t indent_step_ = 0, error_handler_t error_handler_ = error_handler_t::strict) : o(std::move(s)) - , loc(std::localeconv()) - , thousands_sep(loc->thousands_sep == nullptr ? '\0' : std::char_traits::to_char_type(* (loc->thousands_sep))) - , decimal_point(loc->decimal_point == nullptr ? '\0' : std::char_traits::to_char_type(* (loc->decimal_point))) + , locale(std::localeconv()) , indent_char(ichar) - , indent_string(512, indent_char) + , pretty_print(pretty_print_) + , ensure_ascii(ensure_ascii_) + , indent_step(indent_step_) , error_handler(error_handler_) {} @@ -21387,8 +21450,8 @@ class serializer This function is called by the public member function dump and organizes the serialization internally. The indentation level is propagated as - additional parameter. In case of arrays and objects, the function is - called recursively. + additional parameter. Arrays and objects are serialized without recursion, + however deeply they are nested. - strings and object keys are escaped using `escape_string()` - integer numbers are converted implicitly via `operator<<` @@ -21397,89 +21460,109 @@ class serializer byte array @param[in] val value to serialize - @param[in] pretty_print whether the output shall be pretty-printed - @param[in] ensure_ascii If @a ensure_ascii is true, all non-ASCII characters - in the output are escaped with `\uXXXX` sequences, and the result consists - of ASCII characters only. - @param[in] indent_step the indent level @param[in] current_indent the current indent level (only used internally) */ void dump(const BasicJsonType& val, - const bool pretty_print, - const bool ensure_ascii, - const unsigned int indent_step, - const unsigned int current_indent = 0) + const std::size_t current_indent = 0) + { + dump_internal(val, current_indent); + flush(); + } + + JSON_PRIVATE_UNLESS_TESTED: + /*! + @brief worker for @ref dump + + Identical in behavior to the historical @ref dump, but writes into the + serializer's internal @ref write_buffer instead of issuing a virtual call + per token. The public @ref dump wraps this and flushes the buffer once the + top-level value has been serialized. + + Serializing a container descends into its elements, so a value nested deeply + enough used to exhaust the call stack and terminate the process with no + exception to catch. The descent is bounded here: once @ref dump_depth_limit + levels have been entered, @ref dump_iteratively writes out what is left + without the call stack. A value nested less deeply than that - all but a + vanishing minority - is written by exactly the code that always wrote it. + + @sa https://github.com/nlohmann/json/issues/5387 + */ + void dump_internal(const BasicJsonType& val, + const std::size_t current_indent = 0, + const std::size_t depth = 0) { switch (val.m_data.m_type) { case value_t::object: { + if (JSON_HEDLEY_UNLIKELY(depth >= dump_depth_limit())) + { + dump_iteratively(val, current_indent); + return; + } + if (val.m_data.m_value.object->empty()) { - o->write_characters("{}", 2); + put_literal("{}"); return; } if (pretty_print) { - o->write_characters("{\n", 2); + put_literal("{\n"); // variable to hold indentation for recursive calls - const auto new_indent = current_indent + indent_step; - if (JSON_HEDLEY_UNLIKELY(indent_string.size() < new_indent)) - { - indent_string.resize(indent_string.size() * 2, ' '); - } + const auto new_indent = next_indent(current_indent, indent_step); // first n-1 elements auto i = val.m_data.m_value.object->cbegin(); for (std::size_t cnt = 0; cnt < val.m_data.m_value.object->size() - 1; ++cnt, ++i) { - o->write_characters(indent_string.c_str(), new_indent); - o->write_character('\"'); - dump_escaped(i->first, ensure_ascii); - o->write_characters("\": ", 3); - dump(i->second, true, ensure_ascii, indent_step, new_indent); - o->write_characters(",\n", 2); + put_indent(new_indent); + put_char('"'); + dump_escaped(i->first); + put_literal("\": "); + dump_internal(i->second, new_indent, depth + 1); + put_literal(",\n"); } // last element JSON_ASSERT(i != val.m_data.m_value.object->cend()); JSON_ASSERT(std::next(i) == val.m_data.m_value.object->cend()); - o->write_characters(indent_string.c_str(), new_indent); - o->write_character('\"'); - dump_escaped(i->first, ensure_ascii); - o->write_characters("\": ", 3); - dump(i->second, true, ensure_ascii, indent_step, new_indent); + put_indent(new_indent); + put_char('"'); + dump_escaped(i->first); + put_literal("\": "); + dump_internal(i->second, new_indent, depth + 1); - o->write_character('\n'); - o->write_characters(indent_string.c_str(), current_indent); - o->write_character('}'); + put_char('\n'); + put_indent(current_indent); + put_char('}'); } else { - o->write_character('{'); + put_char('{'); // first n-1 elements auto i = val.m_data.m_value.object->cbegin(); for (std::size_t cnt = 0; cnt < val.m_data.m_value.object->size() - 1; ++cnt, ++i) { - o->write_character('\"'); - dump_escaped(i->first, ensure_ascii); - o->write_characters("\":", 2); - dump(i->second, false, ensure_ascii, indent_step, current_indent); - o->write_character(','); + put_char('"'); + dump_escaped(i->first); + put_literal("\":"); + dump_internal(i->second, current_indent, depth + 1); + put_char(','); } // last element JSON_ASSERT(i != val.m_data.m_value.object->cend()); JSON_ASSERT(std::next(i) == val.m_data.m_value.object->cend()); - o->write_character('\"'); - dump_escaped(i->first, ensure_ascii); - o->write_characters("\":", 2); - dump(i->second, false, ensure_ascii, indent_step, current_indent); + put_char('"'); + dump_escaped(i->first); + put_literal("\":"); + dump_internal(i->second, current_indent, depth + 1); - o->write_character('}'); + put_char('}'); } return; @@ -21487,58 +21570,60 @@ class serializer case value_t::array: { + if (JSON_HEDLEY_UNLIKELY(depth >= dump_depth_limit())) + { + dump_iteratively(val, current_indent); + return; + } + if (val.m_data.m_value.array->empty()) { - o->write_characters("[]", 2); + put_literal("[]"); return; } if (pretty_print) { - o->write_characters("[\n", 2); + put_literal("[\n"); // variable to hold indentation for recursive calls - const auto new_indent = current_indent + indent_step; - if (JSON_HEDLEY_UNLIKELY(indent_string.size() < new_indent)) - { - indent_string.resize(indent_string.size() * 2, ' '); - } + const auto new_indent = next_indent(current_indent, indent_step); // first n-1 elements for (auto i = val.m_data.m_value.array->cbegin(); i != val.m_data.m_value.array->cend() - 1; ++i) { - o->write_characters(indent_string.c_str(), new_indent); - dump(*i, true, ensure_ascii, indent_step, new_indent); - o->write_characters(",\n", 2); + put_indent(new_indent); + dump_internal(*i, new_indent, depth + 1); + put_literal(",\n"); } // last element JSON_ASSERT(!val.m_data.m_value.array->empty()); - o->write_characters(indent_string.c_str(), new_indent); - dump(val.m_data.m_value.array->back(), true, ensure_ascii, indent_step, new_indent); + put_indent(new_indent); + dump_internal(val.m_data.m_value.array->back(), new_indent, depth + 1); - o->write_character('\n'); - o->write_characters(indent_string.c_str(), current_indent); - o->write_character(']'); + put_char('\n'); + put_indent(current_indent); + put_char(']'); } else { - o->write_character('['); + put_char('['); // first n-1 elements for (auto i = val.m_data.m_value.array->cbegin(); i != val.m_data.m_value.array->cend() - 1; ++i) { - dump(*i, false, ensure_ascii, indent_step, current_indent); - o->write_character(','); + dump_internal(*i, current_indent, depth + 1); + put_char(','); } // last element JSON_ASSERT(!val.m_data.m_value.array->empty()); - dump(val.m_data.m_value.array->back(), false, ensure_ascii, indent_step, current_indent); + dump_internal(val.m_data.m_value.array->back(), current_indent, depth + 1); - o->write_character(']'); + put_char(']'); } return; @@ -21546,9 +21631,9 @@ class serializer case value_t::string: { - o->write_character('\"'); - dump_escaped(*val.m_data.m_value.string, ensure_ascii); - o->write_character('\"'); + put_char('"'); + dump_escaped(*val.m_data.m_value.string); + put_char('"'); return; } @@ -21556,70 +21641,66 @@ class serializer { if (pretty_print) { - o->write_characters("{\n", 2); + put_literal("{\n"); // variable to hold indentation for recursive calls - const auto new_indent = current_indent + indent_step; - if (JSON_HEDLEY_UNLIKELY(indent_string.size() < new_indent)) - { - indent_string.resize(indent_string.size() * 2, ' '); - } + const auto new_indent = next_indent(current_indent, indent_step); - o->write_characters(indent_string.c_str(), new_indent); + put_indent(new_indent); - o->write_characters("\"bytes\": [", 10); + put_literal("\"bytes\": ["); if (!val.m_data.m_value.binary->empty()) { for (auto i = val.m_data.m_value.binary->cbegin(); i != val.m_data.m_value.binary->cend() - 1; ++i) { - dump_integer(*i); - o->write_characters(", ", 2); + dump_byte(*i); + put_literal(", "); } - dump_integer(val.m_data.m_value.binary->back()); + dump_byte(val.m_data.m_value.binary->back()); } - o->write_characters("],\n", 3); - o->write_characters(indent_string.c_str(), new_indent); + put_literal("],\n"); + put_indent(new_indent); - o->write_characters("\"subtype\": ", 11); + put_literal("\"subtype\": "); if (val.m_data.m_value.binary->has_subtype()) { dump_integer(val.m_data.m_value.binary->subtype()); } else { - o->write_characters("null", 4); + put_literal("null"); } - o->write_character('\n'); - o->write_characters(indent_string.c_str(), current_indent); - o->write_character('}'); + put_char('\n'); + put_indent(current_indent); + put_char('}'); } else { - o->write_characters("{\"bytes\":[", 10); + put_literal("{\"bytes\":["); if (!val.m_data.m_value.binary->empty()) { for (auto i = val.m_data.m_value.binary->cbegin(); i != val.m_data.m_value.binary->cend() - 1; ++i) { - dump_integer(*i); - o->write_character(','); + dump_byte(*i); + put_char(','); } - dump_integer(val.m_data.m_value.binary->back()); + dump_byte(val.m_data.m_value.binary->back()); } - o->write_characters("],\"subtype\":", 12); + put_literal("],\"subtype\":"); if (val.m_data.m_value.binary->has_subtype()) { dump_integer(val.m_data.m_value.binary->subtype()); - o->write_character('}'); + put_char('}'); } else { - o->write_characters("null}", 5); + put_literal("null}"); } } return; @@ -21629,11 +21710,11 @@ class serializer { if (val.m_data.m_value.boolean) { - o->write_characters("true", 4); + put_literal("true"); } else { - o->write_characters("false", 5); + put_literal("false"); } return; } @@ -21658,13 +21739,13 @@ class serializer case value_t::discarded: { - o->write_characters("", 11); + put_literal(""); return; } case value_t::null: { - o->write_characters("null", 4); + put_literal("null"); return; } @@ -21673,6 +21754,367 @@ class serializer } } + private: + /// the number of levels @ref dump_internal descends into before it hands + /// over to @ref dump_iteratively + static constexpr std::size_t dump_depth_limit() + { + return 128; + } + + /*! + @brief write out @a val and everything below it without the call stack + + Emits the same bytes as @ref dump_internal, keeping the containers it has + entered on an explicit stack instead of descending into them. Only reached + for values nested deeper than @ref dump_depth_limit, which is why it is not + written for speed: walking every value this way measured up to 20% slower on + object-heavy documents than letting the compiler drive the descent. + */ + void dump_iteratively(const BasicJsonType& val, + const std::size_t current_indent = 0) + { + // Scalars, empty containers and binary values are written by dump_value + // alone, so nothing is allocated for them: only a container with + // elements is ever pushed. + std::vector stack; + + dump_value(val, current_indent, stack); + + while (!stack.empty()) + { + dump_frame& frame = stack.back(); + + if (frame.value->m_data.m_type == value_t::object) + { + const auto* object = frame.value->m_data.m_value.object; + + if (frame.object_it == object->cend()) + { + if (pretty_print) + { + put_char('\n'); + put_indent(frame.current_indent); + } + + put_char('}'); + stack.pop_back(); + continue; + } + + // the separator goes in front of every element but the first, + // which puts exactly one between each pair and none at the end + if (frame.object_it != object->cbegin()) + { + if (pretty_print) + { + put_literal(",\n"); + } + else + { + put_char(','); + } + } + + if (pretty_print) + { + put_indent(frame.child_indent); + } + + put_char('"'); + dump_escaped(frame.object_it->first); + + if (pretty_print) + { + put_literal("\": "); + } + else + { + put_literal("\":"); + } + + const BasicJsonType& element = frame.object_it->second; + ++frame.object_it; + + // read everything needed from the frame before this: entering a + // container pushes another one and can move them all + const std::size_t element_indent = frame.child_indent; + dump_value(element, element_indent, stack); + } + else + { + const auto* array = frame.value->m_data.m_value.array; + + if (frame.array_it == array->cend()) + { + if (pretty_print) + { + put_char('\n'); + put_indent(frame.current_indent); + } + + put_char(']'); + stack.pop_back(); + continue; + } + + if (frame.array_it != array->cbegin()) + { + if (pretty_print) + { + put_literal(",\n"); + } + else + { + put_char(','); + } + } + + if (pretty_print) + { + put_indent(frame.child_indent); + } + + const BasicJsonType& element = *frame.array_it; + ++frame.array_it; + + // see above + const std::size_t element_indent = frame.child_indent; + dump_value(element, element_indent, stack); + } + } + } + + private: + /// @brief a container that has been opened but not closed yet + struct dump_frame + { + dump_frame(const BasicJsonType* value_, const std::size_t current_indent_, + const std::size_t child_indent_) noexcept + : value(value_) + , current_indent(current_indent_) + , child_indent(child_indent_) + {} + + /// the object or array being serialized + const BasicJsonType* value; + /// the element to serialize next; which of the two is live follows from + /// the type of @a value. They are kept side by side rather than in a + /// union, which would need its special members written out by hand, see + /// detail/iterators/internal_iterator.hpp + typename BasicJsonType::object_t::const_iterator object_it{}; + typename BasicJsonType::array_t::const_iterator array_it{}; + /// the indentation of the container itself, used by its closing bracket + std::size_t current_indent; + /// the indentation of the container's elements + std::size_t child_indent; + }; + + /*! + @brief serialize the value @a val, but not the elements of a container + + An object or array with elements is opened and pushed onto @a stack for + @ref dump_internal to walk; everything else - including a binary value, + which looks like an object but has no elements to descend into - is written + out here in full. + */ + void dump_value(const BasicJsonType& val, + const std::size_t current_indent, + std::vector& stack) + { + switch (val.m_data.m_type) + { + case value_t::object: + { + if (val.m_data.m_value.object->empty()) + { + put_literal("{}"); + return; + } + + std::size_t child_indent = current_indent; + + if (pretty_print) + { + put_literal("{\n"); + child_indent = next_indent(current_indent, indent_step); + } + else + { + put_char('{'); + } + + stack.emplace_back(&val, current_indent, child_indent); + stack.back().object_it = val.m_data.m_value.object->cbegin(); + return; + } + + case value_t::array: + { + if (val.m_data.m_value.array->empty()) + { + put_literal("[]"); + return; + } + + std::size_t child_indent = current_indent; + + if (pretty_print) + { + put_literal("[\n"); + child_indent = next_indent(current_indent, indent_step); + } + else + { + put_char('['); + } + + stack.emplace_back(&val, current_indent, child_indent); + stack.back().array_it = val.m_data.m_value.array->cbegin(); + return; + } + + case value_t::string: + { + put_char('"'); + dump_escaped(*val.m_data.m_value.string); + put_char('"'); + return; + } + + case value_t::binary: + { + if (pretty_print) + { + put_literal("{\n"); + + // variable to hold indentation for the bytes + const auto new_indent = next_indent(current_indent, indent_step); + + put_indent(new_indent); + + put_literal("\"bytes\": ["); + + if (!val.m_data.m_value.binary->empty()) + { + for (auto i = val.m_data.m_value.binary->cbegin(); + i != val.m_data.m_value.binary->cend() - 1; ++i) + { + dump_byte(*i); + put_literal(", "); + } + dump_byte(val.m_data.m_value.binary->back()); + } + + put_literal("],\n"); + put_indent(new_indent); + + put_literal("\"subtype\": "); + if (val.m_data.m_value.binary->has_subtype()) + { + dump_integer(val.m_data.m_value.binary->subtype()); + } + else + { + put_literal("null"); + } + put_char('\n'); + put_indent(current_indent); + put_char('}'); + } + else + { + put_literal("{\"bytes\":["); + + if (!val.m_data.m_value.binary->empty()) + { + for (auto i = val.m_data.m_value.binary->cbegin(); + i != val.m_data.m_value.binary->cend() - 1; ++i) + { + dump_byte(*i); + put_char(','); + } + dump_byte(val.m_data.m_value.binary->back()); + } + + put_literal("],\"subtype\":"); + if (val.m_data.m_value.binary->has_subtype()) + { + dump_integer(val.m_data.m_value.binary->subtype()); + put_char('}'); + } + else + { + put_literal("null}"); + } + } + return; + } + + case value_t::boolean: + { + if (val.m_data.m_value.boolean) + { + put_literal("true"); + } + else + { + put_literal("false"); + } + return; + } + + case value_t::number_integer: + { + dump_integer(val.m_data.m_value.number_integer); + return; + } + + case value_t::number_unsigned: + { + dump_integer(val.m_data.m_value.number_unsigned); + return; + } + + case value_t::number_float: + { + dump_float(val.m_data.m_value.number_float); + return; + } + + case value_t::discarded: + { + put_literal(""); + return; + } + + case value_t::null: + { + put_literal("null"); + return; + } + + default: // LCOV_EXCL_LINE + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE + } + } + + + /*! + @brief the indentation level to use for the children of the current value + + A very large @a indent_step can wrap the unsigned accumulation on deep + nesting, which would silently truncate the indentation. Far harder to reach + now that the accumulator is a std::size_t, but still reachable where that is + 32 bits wide. + */ + static std::size_t next_indent(const std::size_t current_indent, const std::size_t indent_step) + { + const std::size_t new_indent = current_indent + indent_step; + JSON_ASSERT(new_indent >= current_indent); + return new_indent; + } + JSON_PRIVATE_UNLESS_TESTED: /*! @brief dump escaped string @@ -21683,12 +22125,32 @@ class serializer representation. The escaped string is written to output stream @a o. @param[in] s the string to escape - @param[in] ensure_ascii whether to escape non-ASCII characters with - \uXXXX sequences @complexity Linear in the length of string @a s. */ - void dump_escaped(const string_t& s, const bool ensure_ascii) + void dump_escaped(const string_t& s) + { + // dispatch once here rather than test the flag inside the loop: it does + // not change while a string is written, and folding it lets each of the + // two scanners be inlined into a loop of its own + if (ensure_ascii) + { + dump_escaped_impl(s); + } + else + { + dump_escaped_impl(s); + } + } + + /*! + @brief worker for @ref dump_escaped + + @a ensure_ascii is a template parameter here so that the branch on it is + resolved once, outside the loop; see @ref dump_escaped. + */ + template + void dump_escaped_impl(const string_t& s) { std::uint32_t codepoint{}; std::uint8_t state = UTF8_ACCEPT; @@ -21700,6 +22162,56 @@ class serializer for (std::size_t i = 0; i < s.size(); ++i) { + // Fast path: at a character boundary (state == UTF8_ACCEPT), + // bulk-copy the longest run of bytes that need no escaping using a + // SWAR scanner shared with the lexer's contiguous path. The scanner + // stops exactly at the first byte dump_escaped would handle + // individually, so that byte is left to the byte-at-a-time path + // below, keeping escaping output and error diagnostics unchanged. + // + // - EnsureAscii == false: string_bulk_run() copies ordinary bytes + // and complete well-formed UTF-8, stopping at a quote, backslash, + // control character (< 0x20), or ill-formed/truncated sequence. + // - EnsureAscii == true: only printable ASCII may be copied + // verbatim; find_ascii_copyable_run() additionally stops at 0x7F + // and every non-ASCII byte (>= 0x80), which must be \u-escaped. + if (state == UTF8_ACCEPT) + { + const auto* const data = reinterpret_cast(s.data()); + // A run can only be non-empty when the very first byte is one + // the scanner may copy, so test that single byte before paying + // for the scan. Without it, text whose characters all have to be + // escaped - CJK under ensure_ascii, where every byte is >= 0x80 - + // runs the scanner once per character only to be told zero. + std::size_t run = 0; + if (!EnsureAscii) + { + run = string_bulk_run(data + i, s.size() - i); + } + else if (is_ascii_copyable(data[i])) + { + run = find_ascii_copyable_run(data + i, s.size() - i); + } + if (run != 0) + { + // emit any bytes still pending in string_buffer first to + // preserve output order, then write the run directly + if (bytes != 0) + { + put_buffer(string_buffer, bytes); + bytes = 0; + } + put_string(s, i, i + run); + bytes_after_last_accept = 0; + undumped_chars = 0; + i += run; + if (i >= s.size()) + { + break; + } + } + } + const auto byte = static_cast(s[i]); switch (decode(state, codepoint, byte)) @@ -21746,7 +22258,7 @@ class serializer case 0x22: // quotation mark { string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = '\"'; + string_buffer[bytes++] = '"'; break; } @@ -21760,8 +22272,8 @@ class serializer default: { // escape control characters (0x00..0x1F) or, if - // ensure_ascii parameter is used, non-ASCII characters - if ((codepoint <= 0x1F) || (ensure_ascii && (codepoint >= 0x7F))) + // EnsureAscii parameter is used, non-ASCII characters + if ((codepoint <= 0x1F) || (EnsureAscii && (codepoint >= 0x7F))) { if (codepoint <= 0xFFFF) { @@ -21788,7 +22300,7 @@ class serializer // written ("\uxxxx\uxxxx\0") for one code point if (string_buffer.size() - bytes < 13) { - o->write_characters(string_buffer.data(), bytes); + put_buffer(string_buffer, bytes); bytes = 0; } @@ -21826,7 +22338,7 @@ class serializer if (error_handler == error_handler_t::replace) { // add a replacement character - if (ensure_ascii) + if (EnsureAscii) { string_buffer[bytes++] = '\\'; string_buffer[bytes++] = 'u'; @@ -21847,7 +22359,7 @@ class serializer // written ("\uxxxx\uxxxx\0") for one code point if (string_buffer.size() - bytes < 13) { - o->write_characters(string_buffer.data(), bytes); + put_buffer(string_buffer, bytes); bytes = 0; } @@ -21869,7 +22381,7 @@ class serializer default: // decode found yet incomplete multibyte code point { - if (!ensure_ascii) + if (!EnsureAscii) { // code point will not be escaped - copy byte to buffer string_buffer[bytes++] = s[i]; @@ -21886,7 +22398,7 @@ class serializer // write buffer if (bytes > 0) { - o->write_characters(string_buffer.data(), bytes); + put_buffer(string_buffer, bytes); } } else @@ -21902,22 +22414,22 @@ class serializer case error_handler_t::ignore: { // write all accepted bytes - o->write_characters(string_buffer.data(), bytes_after_last_accept); + put_buffer(string_buffer, bytes_after_last_accept); break; } case error_handler_t::replace: { // write all accepted bytes - o->write_characters(string_buffer.data(), bytes_after_last_accept); + put_buffer(string_buffer, bytes_after_last_accept); // add a replacement character - if (ensure_ascii) + if (EnsureAscii) { - o->write_characters("\\ufffd", 6); + put_literal("\\ufffd"); } else { - o->write_characters("\xEF\xBF\xBD", 3); + put_literal("\xEF\xBF\xBD"); } break; } @@ -21928,6 +22440,160 @@ class serializer } } + private: + /*! + @brief append a single character to the write buffer + + Structural characters ('{', '"', ',', ...) previously went straight to the + output adapter, one virtual call each. Buffering them and flushing in bulk + turns those many indirect calls into a single memcpy plus an occasional + flush, which dominates the cost of serializing object/array-heavy values. + */ + void put_char(char c) + { + if (JSON_HEDLEY_UNLIKELY(write_buffer_pos == write_buffer.size())) + { + flush(); + } + write_buffer[write_buffer_pos++] = c; + } + + /*! + @brief append @a indent indentation characters to the write buffer + + Writes the indentation straight into the buffer instead of copying it out of + a pre-grown indentation string, so no auxiliary string has to be sized, + resized, or kept in sync with the deepest nesting level reached. + + An indentation wider than the buffer is emitted by filling the buffer with + the indentation character once and flushing that same content repeatedly: + flushing does not disturb what the buffer holds, so re-filling it between + flushes would be redundant work. + */ + void put_indent(std::size_t indent) + { + // closing braces at the outermost level ask for no indentation at all + if (indent == 0) + { + return; + } + + const std::size_t capacity = write_buffer.size(); + + // fill whatever room is left in the buffer; this is the whole job + // whenever the indentation is narrower than the buffer, which is the + // case for every sane indent_step + const std::size_t head = (std::min)(indent, capacity - write_buffer_pos); + std::memset(write_buffer.data() + write_buffer_pos, indent_char, head); + write_buffer_pos += head; + indent -= head; + + if (JSON_HEDLEY_LIKELY(indent == 0)) + { + return; + } + + // the buffer is full and the remainder spans whole buffer-fulls: flush + // what is pending, then fill the buffer with the indentation character + // exactly once and hand the same bytes to the adapter as often as needed + flush(); + std::memset(write_buffer.data(), indent_char, capacity); + + while (indent >= capacity) + { + write_buffer_pos = capacity; + flush(); + indent -= capacity; + } + + // the buffer still holds indentation characters throughout, so the tail + // only has to be claimed, not written again + write_buffer_pos = indent; + } + + /*! + @brief append a string literal to the write buffer + + The length comes from the array bound rather than a hand-written count, so + it cannot drift out of sync with the literal. A literal always fits into the + buffer (checked at compile time), so unlike @ref put_string this needs no + write-through path for oversized runs. + */ + template + void put_literal(const char (&s)[N]) // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + { + static_assert(N >= 2, "put_literal expects a non-empty string literal"); + // the array bound counts the terminating NUL, which is not written + constexpr std::size_t length = N - 1; + static_assert(length < write_buffer_size, "string literal must fit into the write buffer"); + + if (JSON_HEDLEY_UNLIKELY(write_buffer_pos + length > write_buffer.size())) + { + flush(); + } + std::memcpy(write_buffer.data() + write_buffer_pos, s, length); + write_buffer_pos += length; + } + + /*! + @brief append the characters of @a str in [@a start, @a end) + + The only way to append a run of characters: @a str carries its own bound, + so the range can be checked against it, which a bare pointer plus a count + could not do. Runs that do not fit the buffer are written straight through + the output adapter (after flushing what is pending), so large string and + number payloads are not copied an extra time. + */ + template + void put_string(const StringType& str, std::size_t start, std::size_t end) + { + JSON_ASSERT(start <= end); + JSON_ASSERT(end <= str.size()); + + const char* const s = str.data() + start; + const std::size_t length = end - start; + + if (JSON_HEDLEY_UNLIKELY(length >= write_buffer.size())) + { + flush(); + o->write_characters(s, length); + return; + } + if (JSON_HEDLEY_UNLIKELY(write_buffer_pos + length > write_buffer.size())) + { + flush(); + } + std::memcpy(write_buffer.data() + write_buffer_pos, s, length); + write_buffer_pos += length; + } + + /*! + @brief append the first @a length characters of a fixed-size buffer + */ + template + void put_buffer(const std::array& buffer, std::size_t length) + { + put_string(buffer, 0, length); + } + + JSON_PRIVATE_UNLESS_TESTED: + /*! + @brief flush the write buffer to the output adapter + + Writing zero characters is a well-defined no-op for every output adapter, so + the buffered length is passed through unconditionally (no empty-guard branch + to leave uncovered). + + @note dump_escaped() and dump_integer()/dump_float() write into the internal + write buffer; callers that invoke them directly (rather than through the + public dump()) must call flush() before inspecting the output. + */ + void flush() + { + o->write_characters(write_buffer.data(), write_buffer_pos); + write_buffer_pos = 0; + } + private: /*! @brief count digits @@ -22016,6 +22682,62 @@ class serializer return false; } + /*! + @brief write the decimal representation of the byte @a value + + A binary value's bytes are always in [0, 255], so writing one needs neither + the digit counting nor the 64-bit arithmetic that @ref dump_integer does for + an arbitrary number, and the three digits it takes at most are written + straight into the write buffer. + + Any byte type that is not a plain unsigned byte is left to @ref dump_integer, + whose representation of it may differ. + */ + template + void dump_byte(const ByteType value) + { + dump_byte(value, std::integral_constant < bool, + std::is_unsigned::value && sizeof(ByteType) == 1 + && !std::is_same::value > {}); + } + + template + void dump_byte(const ByteType value, std::false_type /*is_plain_byte*/) + { + dump_integer(value); + } + + template + void dump_byte(const ByteType value, std::true_type /*is_plain_byte*/) + { + if (JSON_HEDLEY_UNLIKELY(write_buffer_pos + 3 > write_buffer.size())) + { + flush(); + } + + const auto byte = static_cast(value); + // Accumulate the offset in a local and store it back once. Writing + // through write_buffer[] is a char write, which may alias any object, + // so with the member updated in place the compiler has to reload and + // store it around every digit - measured 2.4x slower on a dump of a + // multi-megabyte binary value. + std::size_t pos = write_buffer_pos; + + if (byte >= 100) + { + write_buffer[pos++] = static_cast('0' + (byte / 100)); + write_buffer[pos++] = static_cast('0' + ((byte / 10) % 10)); + } + else if (byte >= 10) + { + write_buffer[pos++] = static_cast('0' + (byte / 10)); + } + + write_buffer[pos++] = static_cast('0' + (byte % 10)); + + write_buffer_pos = pos; + } + /*! @brief dump an integer @@ -22052,7 +22774,7 @@ class serializer // special case for "0" if (x == 0) { - o->write_character('0'); + put_char('0'); return; } @@ -22105,7 +22827,7 @@ class serializer *(--buffer_ptr) = static_cast('0' + abs_value); } - o->write_characters(number_buffer.data(), n_chars); + put_buffer(number_buffer, n_chars); } /*! @@ -22121,7 +22843,7 @@ class serializer // NaN / inf if (!std::isfinite(x)) { - o->write_characters("null", 4); + put_literal("null"); return; } @@ -22142,7 +22864,7 @@ class serializer auto* begin = number_buffer.data(); auto* end = ::nlohmann::detail::to_chars(begin, begin + number_buffer.size(), x); - o->write_characters(begin, static_cast(end - begin)); + put_buffer(number_buffer, static_cast(end - begin)); } JSON_HEDLEY_NON_NULL(1) @@ -22173,27 +22895,27 @@ class serializer JSON_ASSERT(static_cast(len) < number_buffer.size()); // erase thousands separators - if (thousands_sep != '\0') + if (locale.thousands_sep != '\0') { // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::remove returns an iterator, see https://github.com/nlohmann/json/issues/3081 - const auto end = std::remove(number_buffer.begin(), number_buffer.begin() + len, thousands_sep); + const auto end = std::remove(number_buffer.begin(), number_buffer.begin() + len, locale.thousands_sep); std::fill(end, number_buffer.end(), '\0'); JSON_ASSERT((end - number_buffer.begin()) <= len); len = (end - number_buffer.begin()); } // convert decimal point to '.' - if (decimal_point != '\0' && decimal_point != '.') + if (locale.decimal_point != '\0' && locale.decimal_point != '.') { // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::find returns an iterator, see https://github.com/nlohmann/json/issues/3081 - const auto dec_pos = std::find(number_buffer.begin(), number_buffer.end(), decimal_point); + const auto dec_pos = std::find(number_buffer.begin(), number_buffer.end(), locale.decimal_point); if (dec_pos != number_buffer.end()) { *dec_pos = '.'; } } - o->write_characters(number_buffer.data(), static_cast(len)); + put_buffer(number_buffer, static_cast(len)); // determine if we need to append ".0" const bool value_is_int_like = @@ -22205,7 +22927,7 @@ class serializer if (value_is_int_like) { - o->write_characters(".0", 2); + put_literal(".0"); } } @@ -22292,29 +23014,53 @@ class serializer } private: + /// the locale's thousand separator and decimal point characters + struct locale_chars + { + explicit locale_chars(const std::lconv* loc) noexcept + : thousands_sep(loc->thousands_sep == nullptr ? '\0' : std::char_traits::to_char_type(* (loc->thousands_sep))) + , decimal_point(loc->decimal_point == nullptr ? '\0' : std::char_traits::to_char_type(* (loc->decimal_point))) + {} + + const char thousands_sep; + const char decimal_point; + }; + /// the output of the serializer output_adapter_t o = nullptr; /// a (hopefully) large enough character buffer std::array number_buffer{{}}; - /// the locale - const std::lconv* loc = nullptr; - /// the locale's thousand separator character - const char thousands_sep = '\0'; - /// the locale's decimal point character - const char decimal_point = '\0'; + /// computed once from std::localeconv() at construction; @ref + /// locale_chars keeps std::localeconv()'s pointer from having to be held + /// past the constructor, while still letting these stay const + const locale_chars locale; /// string buffer std::array string_buffer{{}}; /// the indentation character const char indent_char; - /// the indentation string - string_t indent_string; + + /// whether to pretty-print the output + const bool pretty_print; + + /// whether to escape non-ASCII characters with \uXXXX sequences + const bool ensure_ascii; + + /// the indent level + const std::size_t indent_step; /// error_handler how to react on decoding errors const error_handler_t error_handler; + + /// buffer collecting output before it is flushed to the output adapter, so + /// that the many small structural writes become few bulk writes + static constexpr std::size_t write_buffer_size = 1024; + std::array write_buffer{{}}; + /// number of valid bytes currently held in @ref write_buffer + std::size_t write_buffer_pos = 0; }; } // namespace detail @@ -23992,15 +24738,18 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const error_handler_t error_handler = error_handler_t::strict) const { string_t result; - serializer s(detail::output_adapter(result), indent_char, error_handler); if (indent >= 0) { - s.dump(*this, true, ensure_ascii, static_cast(indent)); + serializer s(detail::output_adapter(result), indent_char, + true, ensure_ascii, static_cast(indent), error_handler); + s.dump(*this); } else { - s.dump(*this, false, ensure_ascii, 0); + serializer s(detail::output_adapter(result), indent_char, + false, ensure_ascii, 0, error_handler); + s.dump(*this); } return result; @@ -26729,8 +27478,9 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec o.width(0); // do the actual serialization - serializer s(detail::output_adapter(o), o.fill()); - s.dump(j, pretty_print, false, static_cast(indentation)); + serializer s(detail::output_adapter(o), o.fill(), + pretty_print, false, static_cast(indentation)); + s.dump(j); return o; } diff --git a/tests/src/unit-convenience.cpp b/tests/src/unit-convenience.cpp index 037a4e589..266497867 100644 --- a/tests/src/unit-convenience.cpp +++ b/tests/src/unit-convenience.cpp @@ -98,8 +98,9 @@ void check_escaped(const char* original, const char* escaped = "", bool ensure_a void check_escaped(const char* original, const char* escaped, const bool ensure_ascii) { std::stringstream ss; - json::serializer s(nlohmann::detail::output_adapter(ss), ' '); - s.dump_escaped(original, ensure_ascii); + json::serializer s(nlohmann::detail::output_adapter(ss), ' ', false, ensure_ascii); + s.dump_escaped(original); + s.flush(); // dump_escaped writes into the serializer's internal buffer CHECK(ss.str() == escaped); } } // namespace diff --git a/tests/src/unit-serialization.cpp b/tests/src/unit-serialization.cpp index caf720671..eddf59f2c 100644 --- a/tests/src/unit-serialization.cpp +++ b/tests/src/unit-serialization.cpp @@ -387,3 +387,232 @@ TEST_CASE("dump for basic_json with long double number_float_t") check_same(100.0L, 100.0); } } + +TEST_CASE("serialization of strings (bulk fast path)") +{ + // These cases exercise the SWAR bulk-copy fast path in dump_escaped and the + // internal write buffer: long runs, escapes interrupting runs, 0x7F/DEL, + // multibyte UTF-8 under both ensure_ascii settings, and payloads larger than + // the write buffer. + + SECTION("long unescaped ASCII exceeds the write buffer") + { + const std::string big(3000, 'a'); + const json j = big; + CHECK(j.dump() == '"' + big + '"'); + CHECK(j.dump(-1, ' ', true) == '"' + big + '"'); + // round-trips + CHECK(json::parse(j.dump()) == j); + } + + SECTION("runs interrupted by escapes") + { + const json j = std::string(500, 'x') + "\n\"\\" + std::string(500, 'y'); + const std::string out = j.dump(); + CHECK(out == '"' + std::string(500, 'x') + "\\n\\\"\\\\" + std::string(500, 'y') + '"'); + CHECK(json::parse(out) == j); + } + + SECTION("DEL (0x7F) depends on ensure_ascii") + { + const json j = std::string("a\x7f" "b"); + CHECK(j.dump(-1, ' ', false) == "\"a\x7f" "b\""); // copied verbatim + CHECK(j.dump(-1, ' ', true) == "\"a\\u007fb\""); // escaped + } + + SECTION("multibyte UTF-8 under both ensure_ascii settings") + { + const json j = std::string("A\xc3\xa9\xe4\xbd\xa0\xf0\x9f\x98\x80Z"); // A é 你 😀 Z + // not escaping non-ASCII: bytes are copied through the bulk validator + CHECK(j.dump(-1, ' ', false) == "\"A\xc3\xa9\xe4\xbd\xa0\xf0\x9f\x98\x80Z\""); + // ensure_ascii: escaped (with a surrogate pair for the emoji) + CHECK(j.dump(-1, ' ', true) == "\"A\\u00e9\\u4f60\\ud83d\\ude00Z\""); + CHECK(json::parse(j.dump(-1, ' ', true)) == j); + } + + SECTION("many small structural writes exceed the write buffer") + { + json arr = json::array(); + for (int i = 0; i < 2000; ++i) + { + arr.push_back(i); + } + const std::string out = arr.dump(); + CHECK(out.front() == '['); + CHECK(out.back() == ']'); + CHECK(json::parse(out) == arr); + + json obj = json::object(); + for (int i = 0; i < 500; ++i) + { + obj["key" + std::to_string(i)] = i; + } + CHECK(json::parse(obj.dump()) == obj); + CHECK(json::parse(obj.dump(2)) == obj); + + // an array of many empty strings emits a long run of single-character + // writes ('"', '"', ',') at shallow nesting depth, so the write buffer + // fills and flushes mid-run without the deep recursion that would + // overflow the stack on some debug builds + json many_empty = json::array(); + for (int i = 0; i < 500; ++i) + { + many_empty.push_back(""); + } + const std::string out2 = many_empty.dump(); + CHECK(out2.size() > 1024); // spans multiple write-buffer flushes + CHECK(out2.front() == '['); + CHECK(out2.back() == ']'); + CHECK(json::parse(out2) == many_empty); + } + + SECTION("invalid UTF-8 handling is unaffected by the fast path") + { + const json j = std::string("valid\xff" "more"); + CHECK_THROWS_WITH_AS(j.dump(), "[json.exception.type_error.316] invalid UTF-8 byte at index 5: 0xFF", json::type_error&); + CHECK(j.dump(-1, ' ', false, json::error_handler_t::replace) == "\"valid\xef\xbf\xbd" "more\""); + CHECK(j.dump(-1, ' ', true, json::error_handler_t::replace) == "\"valid\\ufffdmore\""); + CHECK(j.dump(-1, ' ', false, json::error_handler_t::ignore) == "\"validmore\""); + } +} + +TEST_CASE("indentation is written straight into the write buffer") +{ + // put_indent() memsets the indentation into the write buffer instead of + // copying it out of a pre-grown indentation string. These cases cover an + // indentation wider than the buffer, a non-space indentation character, and + // nesting deep enough that the accumulated indentation spans several + // buffer-fulls - the situations the old grow-a-string approach got wrong. + + SECTION("indent_step wider than the write buffer") + { + const json j = {{"a", 1}}; + // 2000 > the 1024-byte write buffer, and > the 512 the indentation + // string used to start at + CHECK(j.dump(2000) == "{\n" + std::string(2000, ' ') + "\"a\": 1\n}"); + // several whole buffer-fulls, so the buffer is refilled once and then + // flushed repeatedly + CHECK(j.dump(5000) == "{\n" + std::string(5000, ' ') + "\"a\": 1\n}"); + CHECK(j.dump(5000, '\t') == "{\n" + std::string(5000, '\t') + "\"a\": 1\n}"); + // an exact multiple of the buffer size + CHECK(j.dump(4096) == "{\n" + std::string(4096, ' ') + "\"a\": 1\n}"); + } + + SECTION("a non-space indentation character is used throughout") + { + const json j = {{"a", 1}}; + // 600 is past the point where the indentation used to be grown, which + // is where a hard-coded space would have shown up + CHECK(j.dump(600, '\t') == "{\n" + std::string(600, '\t') + "\"a\": 1\n}"); + CHECK(j.dump(3, '.') == "{\n...\"a\": 1\n}"); + } + + SECTION("accumulated indentation spans several buffer-fulls") + { + // five levels deep at 400 per level: the innermost value is indented by + // 2000 characters, reached in steps that each straddle the buffer end + json j = json::array({1}); + for (int i = 0; i < 4; ++i) + { + j = json::array({j}); + } + + const std::string out = j.dump(400); + CHECK(out.find(std::string("\n") + std::string(2000, ' ') + "1\n") != std::string::npos); + CHECK(json::parse(out) == j); + } + + SECTION("indentation is unchanged for ordinary widths") + { + const json j = {{"a", {1, 2}}, {"b", nullptr}}; + CHECK(j.dump(2) == "{\n \"a\": [\n 1,\n 2\n ],\n \"b\": null\n}"); + CHECK(j.dump(0) == "{\n\"a\": [\n1,\n2\n],\n\"b\": null\n}"); + } +} + +TEST_CASE("serialization of deeply nested values") +{ + // dump() descends into a bounded number of levels and writes out whatever + // is nested deeper than that without the call stack; see + // https://github.com/nlohmann/json/issues/5387 + + SECTION("nested deeper than the call stack could follow") + { + // parsing is iterative, so building these costs little + const std::size_t depth = 100000; + + const std::string array_text = std::string(depth, '[') + '0' + std::string(depth, ']'); + CHECK(json::parse(array_text).dump() == array_text); + + std::string object_text; + object_text.reserve((6 * depth) + 1); + for (std::size_t i = 0; i < depth; ++i) + { + object_text += "{\"a\":"; + } + object_text += '1'; + object_text.append(depth, '}'); + CHECK(json::parse(object_text).dump() == object_text); + } + + SECTION("depths around the bound of the descent") + { + // Cover every depth around the bound, so that the two ways of writing a + // value are known to meet cleanly - wherever the bound is set. + for (std::size_t d = 1; d <= 300; ++d) + { + CAPTURE(d); + + const std::string array_text = std::string(d, '[') + '7' + std::string(d, ']'); + CHECK(json::parse(array_text).dump() == array_text); + + std::string object_text; + for (std::size_t i = 0; i < d; ++i) + { + object_text += "{\"k\":"; + } + object_text += '7'; + object_text.append(d, '}'); + CHECK(json::parse(object_text).dump() == object_text); + } + } + + SECTION("pretty-printing across the bound") + { + for (std::size_t d = 120; d <= 140; ++d) + { + CAPTURE(d); + + const json j = json::parse(std::string(d, '[') + '7' + std::string(d, ']')); + + std::string expected; + for (std::size_t i = 0; i < d; ++i) + { + expected += std::string(2 * i, ' ') + "[\n"; + } + expected += std::string(2 * d, ' ') + '7'; + for (std::size_t i = d; i > 0; --i) + { + expected += '\n' + std::string(2 * (i - 1), ' ') + ']'; + } + + CHECK(j.dump(2) == expected); + } + } + + SECTION("an empty container below the bound") + { + // an empty container is written out in full and never descended into, + // so it must not gain a newline when it is reached iteratively + for (std::size_t d = 125; d <= 135; ++d) + { + CAPTURE(d); + + const std::string compact = std::string(d, '[') + "[]" + std::string(d, ']'); + CHECK(json::parse(compact).dump() == compact); + + const std::string with_object = std::string(d, '[') + "{}" + std::string(d, ']'); + CHECK(json::parse(with_object).dump() == with_object); + } + } +} From da42a627dc81ba90b790bf3b7a35c7d6780205b5 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Fri, 11 Sep 2026 08:22:50 +0200 Subject: [PATCH 07/18] Stop dump() from heap-allocating its output adapter per call (#5449) * Stop dump() from heap-allocating its output adapter per call The serializer held its output sink as output_adapter_t (a std::shared_ptr>), which dump() and operator<< built via make_shared -- one heap allocation per call for a sink that only wraps a reference to the caller's string or stream. Hold the sink as a non-owning output_adapter_protocol* instead and construct the concrete adapter on the stack at the call site. The write path (o->write_characters) is unchanged, so output is byte-for-byte identical; a compact dump() of a small object drops from 2 heap allocations to 1 (only the returned string remains), ~3% faster. Completes the per-call allocation cleanup on this branch, which already removed the indent_string buffer (both were reported in #5413). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01L1oJ2ggRHS37zeVe94QTA1 Signed-off-by: Claude * Take the output adapter by reference at the serializer ctor Per review: the serializer still holds the adapter as a non-owning pointer, but the constructor now takes output_adapter_protocol& and takes its address internally, so every call site passes a reference. A reference cannot be null and reads as a borrow, which makes the lifetime contract harder to get wrong than handing over a raw pointer. The stored member and the write path are unchanged. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01XAYM1qhSA2FDaDcGfPW3fG Signed-off-by: Niels Lohmann --------- Signed-off-by: Claude Signed-off-by: Niels Lohmann Co-authored-by: Claude --- include/nlohmann/detail/output/serializer.hpp | 11 ++++++----- include/nlohmann/json.hpp | 8 +++++--- single_include/nlohmann/json.hpp | 19 +++++++++++-------- tests/src/unit-convenience.cpp | 3 ++- 4 files changed, 24 insertions(+), 17 deletions(-) diff --git a/include/nlohmann/detail/output/serializer.hpp b/include/nlohmann/detail/output/serializer.hpp index 3dd9162df..9560729ad 100644 --- a/include/nlohmann/detail/output/serializer.hpp +++ b/include/nlohmann/detail/output/serializer.hpp @@ -62,7 +62,8 @@ class serializer public: /*! - @param[in] s output stream to serialize to + @param[in] s output adapter to serialize to; not owned by the serializer, + so it must outlive it (it lives at the call site) @param[in] ichar indentation character to use @param[in] pretty_print_ whether the output shall be pretty-printed @param[in] ensure_ascii_ If @a ensure_ascii_ is true, all non-ASCII @@ -76,12 +77,12 @@ class serializer being threaded through every call to @ref dump, @ref dump_internal and @ref dump_iteratively. */ - serializer(output_adapter_t s, const char ichar, + serializer(output_adapter_protocol& s, const char ichar, const bool pretty_print_ = false, const bool ensure_ascii_ = false, const std::size_t indent_step_ = 0, error_handler_t error_handler_ = error_handler_t::strict) - : o(std::move(s)) + : o(&s) , locale(std::localeconv()) , indent_char(ichar) , pretty_print(pretty_print_) @@ -1678,8 +1679,8 @@ class serializer const char decimal_point; }; - /// the output of the serializer - output_adapter_t o = nullptr; + /// the output of the serializer (non-owning; the adapter lives at the call site) + output_adapter_protocol* o = nullptr; /// a (hopefully) large enough character buffer std::array number_buffer{{}}; diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 7243c3c44..016a4dd27 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -1343,16 +1343,17 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const error_handler_t error_handler = error_handler_t::strict) const { string_t result; + detail::output_string_adapter string_adapter(result); if (indent >= 0) { - serializer s(detail::output_adapter(result), indent_char, + serializer s(string_adapter, indent_char, true, ensure_ascii, static_cast(indent), error_handler); s.dump(*this); } else { - serializer s(detail::output_adapter(result), indent_char, + serializer s(string_adapter, indent_char, false, ensure_ascii, 0, error_handler); s.dump(*this); } @@ -4083,7 +4084,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec o.width(0); // do the actual serialization - serializer s(detail::output_adapter(o), o.fill(), + detail::output_stream_adapter stream_adapter(o); + serializer s(stream_adapter, o.fill(), pretty_print, false, static_cast(indentation)); s.dump(j); return o; diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 8b1f4506e..c94c477ed 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -21410,7 +21410,8 @@ class serializer public: /*! - @param[in] s output stream to serialize to + @param[in] s output adapter to serialize to; not owned by the serializer, + so it must outlive it (it lives at the call site) @param[in] ichar indentation character to use @param[in] pretty_print_ whether the output shall be pretty-printed @param[in] ensure_ascii_ If @a ensure_ascii_ is true, all non-ASCII @@ -21424,12 +21425,12 @@ class serializer being threaded through every call to @ref dump, @ref dump_internal and @ref dump_iteratively. */ - serializer(output_adapter_t s, const char ichar, + serializer(output_adapter_protocol& s, const char ichar, const bool pretty_print_ = false, const bool ensure_ascii_ = false, const std::size_t indent_step_ = 0, error_handler_t error_handler_ = error_handler_t::strict) - : o(std::move(s)) + : o(&s) , locale(std::localeconv()) , indent_char(ichar) , pretty_print(pretty_print_) @@ -23026,8 +23027,8 @@ class serializer const char decimal_point; }; - /// the output of the serializer - output_adapter_t o = nullptr; + /// the output of the serializer (non-owning; the adapter lives at the call site) + output_adapter_protocol* o = nullptr; /// a (hopefully) large enough character buffer std::array number_buffer{{}}; @@ -24738,16 +24739,17 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const error_handler_t error_handler = error_handler_t::strict) const { string_t result; + detail::output_string_adapter string_adapter(result); if (indent >= 0) { - serializer s(detail::output_adapter(result), indent_char, + serializer s(string_adapter, indent_char, true, ensure_ascii, static_cast(indent), error_handler); s.dump(*this); } else { - serializer s(detail::output_adapter(result), indent_char, + serializer s(string_adapter, indent_char, false, ensure_ascii, 0, error_handler); s.dump(*this); } @@ -27478,7 +27480,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec o.width(0); // do the actual serialization - serializer s(detail::output_adapter(o), o.fill(), + detail::output_stream_adapter stream_adapter(o); + serializer s(stream_adapter, o.fill(), pretty_print, false, static_cast(indentation)); s.dump(j); return o; diff --git a/tests/src/unit-convenience.cpp b/tests/src/unit-convenience.cpp index 266497867..0ba57d846 100644 --- a/tests/src/unit-convenience.cpp +++ b/tests/src/unit-convenience.cpp @@ -98,7 +98,8 @@ void check_escaped(const char* original, const char* escaped = "", bool ensure_a void check_escaped(const char* original, const char* escaped, const bool ensure_ascii) { std::stringstream ss; - json::serializer s(nlohmann::detail::output_adapter(ss), ' ', false, ensure_ascii); + nlohmann::detail::output_stream_adapter adapter(ss); + json::serializer s(adapter, ' ', false, ensure_ascii); s.dump_escaped(original); s.flush(); // dump_escaped writes into the serializer's internal buffer CHECK(ss.str() == escaped); From 471584616f98c9273355f85bf3274208c49eafca Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Fri, 11 Sep 2026 08:22:50 +0200 Subject: [PATCH 08/18] Benchmark parsing of pretty-printed JSON (#5456) Every input in the benchmark corpus is minified or only lightly spaced, so none of them exercise the lexer's whitespace handling. Real-world JSON is frequently indented - configuration files, pretty-printed API responses, anything kept under version control - where the insignificant whitespace can outweigh the data itself. Add a ParseIndented family that re-serializes each document with an indentation and parses that. The content is identical to the matching ParseString row, so the pair isolates the cost of the whitespace alone. Measured here, parsing the indented form costs 16-34% more than the minified form of the same document, which nothing in the suite currently reports. Signed-off-by: Niels Lohmann --- tests/benchmarks/src/benchmarks.cpp | 38 +++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tests/benchmarks/src/benchmarks.cpp b/tests/benchmarks/src/benchmarks.cpp index 4de238272..9522df5a4 100644 --- a/tests/benchmarks/src/benchmarks.cpp +++ b/tests/benchmarks/src/benchmarks.cpp @@ -81,6 +81,44 @@ BENCHMARK_CAPTURE(ParseString, signed_ints, TEST_DATA_DIRECTORY "/regressi BENCHMARK_CAPTURE(ParseString, unsigned_ints, TEST_DATA_DIRECTORY "/regression/unsigned_ints.json"); BENCHMARK_CAPTURE(ParseString, small_signed_ints, TEST_DATA_DIRECTORY "/regression/small_signed_ints.json"); +////////////////////////////////////////////////////////////////////////////// +// parse pretty-printed JSON from string +// +// Every file in the corpus above is minified or only lightly spaced, so none of +// them exercise the lexer's whitespace handling. Real-world JSON is frequently +// indented - configuration files, pretty-printed API responses, anything kept +// under version control - where insignificant whitespace can outweigh the data. +// Re-serializing a document with an indentation and parsing that keeps the +// content identical to the ParseString row above, so the pair isolates the cost +// of the whitespace alone. +////////////////////////////////////////////////////////////////////////////// + +static void ParseIndented(benchmark::State& state, const char* filename, int indent) +{ + std::ifstream f(filename); + std::string str((std::istreambuf_iterator(f)), std::istreambuf_iterator()); + const std::string indented = json::parse(str).dump(indent); + + while (state.KeepRunning()) + { + state.PauseTiming(); + auto* j = new json(); + state.ResumeTiming(); + + *j = json::parse(indented); + + state.PauseTiming(); + delete j; + state.ResumeTiming(); + } + + state.SetBytesProcessed(state.iterations() * indented.size()); +} +BENCHMARK_CAPTURE(ParseIndented, jeopardy / 4, TEST_DATA_DIRECTORY "/jeopardy/jeopardy.json", 4); +BENCHMARK_CAPTURE(ParseIndented, canada / 4, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json", 4); +BENCHMARK_CAPTURE(ParseIndented, citm_catalog / 4, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json", 4); +BENCHMARK_CAPTURE(ParseIndented, twitter / 4, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json", 4); + ////////////////////////////////////////////////////////////////////////////// // serialize JSON ////////////////////////////////////////////////////////////////////////////// From 1dc1d09fc66b3df34a806f7d86ba033117162866 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Fri, 11 Sep 2026 08:22:50 +0200 Subject: [PATCH 09/18] Do not search a container for the value the callback rejected (#5457) When a parser callback rejects a value, the placeholder stored for it has to be removed from its parent again. remove_discarded_value() found it by scanning the parent from the beginning, so filtering a container cost one scan per rejected member - quadratic in the number of members of a single container. A rejected value can only ever be the one most recently added to its parent: the last element of an array, or the placeholder key() stored under the current key in an object. Record that key alongside the existing key_keep_stack, and for a container record it again alongside ref_stack so end_object()/end_array() can find it in the parent. Removal is then O(1) for an array and O(log n) for an object, and finding nothing there means nothing was stored, so there is nothing to remove. The key for a container is read before handle_value() may consume it, so it is also correct when the callback rejects the container at its start event and it never reaches its parent at all. Discarding half the members of one object, before -> after: members value rejected container rejected at start 16 000 392 ms -> 3.7 ms 803 ms -> 8.2 ms 64 000 6238 ms -> 14.6 ms 12651 ms -> 32.2 ms 128 000 25339 ms -> 30.6 ms Results are unchanged: 48 000 randomized documents parsed under 12 different filtering callbacks - covering duplicate keys, empty keys, rejected keys and containers rejected at both their start and end events - produce byte identical output before and after. Signed-off-by: Niels Lohmann --- include/nlohmann/detail/input/json_sax.hpp | 87 +++++++++++++++++++--- single_include/nlohmann/json.hpp | 87 +++++++++++++++++++--- tests/src/unit-class_parser.cpp | 52 +++++++++++++ 3 files changed, 208 insertions(+), 18 deletions(-) diff --git a/include/nlohmann/detail/input/json_sax.hpp b/include/nlohmann/detail/input/json_sax.hpp index 8a98ee728..684fac047 100644 --- a/include/nlohmann/detail/input/json_sax.hpp +++ b/include/nlohmann/detail/input/json_sax.hpp @@ -548,6 +548,11 @@ class json_sax_dom_callback_parser const bool keep = callback(static_cast(ref_stack.size()), parse_event_t::object_start, discarded); keep_stack.push_back(keep); + // the key this object will be stored under, read before handle_value() + // may consume it; kept in lockstep with ref_stack so end_object() can + // find the object in its parent again + container_key_stack.push_back(current_key()); + auto val = handle_value(BasicJsonType::value_t::object, true); ref_stack.push_back(val.second); @@ -581,6 +586,9 @@ class json_sax_dom_callback_parser // check callback for the key const bool keep = callback(static_cast(ref_stack.size()), parse_event_t::key, k); key_keep_stack.push_back(keep); + // remember the key so a rejected value can be erased without searching + // the object for it (kept in lockstep with key_keep_stack) + key_stack.push_back(val); // add discarded value at the given key and store the reference for later if (keep && ref_stack.back()) @@ -622,13 +630,16 @@ class json_sax_dom_callback_parser JSON_ASSERT(!ref_stack.empty()); JSON_ASSERT(!keep_stack.empty()); + JSON_ASSERT(!container_key_stack.empty()); ref_stack.pop_back(); keep_stack.pop_back(); + const string_t object_key = std::move(container_key_stack.back()); + container_key_stack.pop_back(); if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_structured()) { // remove discarded value - remove_discarded_value(*ref_stack.back()); + remove_discarded_value(*ref_stack.back(), object_key); } return true; @@ -639,6 +650,9 @@ class json_sax_dom_callback_parser const bool keep = callback(static_cast(ref_stack.size()), parse_event_t::array_start, discarded); keep_stack.push_back(keep); + // see start_object() + container_key_stack.push_back(current_key()); + auto val = handle_value(BasicJsonType::value_t::array, true); ref_stack.push_back(val.second); @@ -701,8 +715,11 @@ class json_sax_dom_callback_parser JSON_ASSERT(!ref_stack.empty()); JSON_ASSERT(!keep_stack.empty()); + JSON_ASSERT(!container_key_stack.empty()); ref_stack.pop_back(); keep_stack.pop_back(); + const string_t object_key = std::move(container_key_stack.back()); + container_key_stack.pop_back(); // remove discarded value if (!ref_stack.empty() && ref_stack.back()) @@ -716,7 +733,7 @@ class json_sax_dom_callback_parser // the array is either still stored under its key or was never // stored, leaving the placeholder key() wrote; both show up as // a discarded member of the parent object - remove_discarded_value(*ref_stack.back()); + remove_discarded_value(*ref_stack.back(), object_key); } } @@ -809,15 +826,56 @@ class json_sax_dom_callback_parser } #endif - /// remove the discarded value the callback rejected from its parent - static void remove_discarded_value(BasicJsonType& parent) + /*! + @brief the key the value now being handled will be stored under + + Empty unless the enclosing container is an object, in which case it is the + key of the pending key() event. Read before handle_value() consumes that + key, so it is also correct when the value never reaches its parent. + */ + string_t current_key() const { - for (auto it = parent.begin(); it != parent.end(); ++it) + if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_object() + && !key_stack.empty()) { - if (it->is_discarded()) + return key_stack.back(); + } + return string_t{}; + } + + /*! + @brief remove the discarded value the callback rejected from its parent + + A rejected value can only ever be the one most recently added to @a parent: + the last element of an array, or the placeholder key() stored under @a key + in an object. Looking there directly makes this O(1) resp. O(log n), where + searching @a parent for it made a filtering parse quadratic in the number of + members of a single container. + + Finding no discarded value there means none was stored in the first place - + the callback rejected the value before it reached its parent - so there is + nothing to remove. + + @param[in,out] parent the container to remove the rejected value from + @param[in] key the key the value was stored under; unused for arrays + */ + static void remove_discarded_value(BasicJsonType& parent, const string_t& key) + { + if (parent.is_array()) + { + auto& array = *parent.m_data.m_value.array; + if (!array.empty() && array.back().is_discarded()) { - parent.erase(it); - break; + array.pop_back(); + } + } + else if (parent.is_object()) + { + auto& object = *parent.m_data.m_value.object; + const auto it = object.find(key); + if (it != object.end() && it->second.is_discarded()) + { + object.erase(it); } } } @@ -867,11 +925,14 @@ class json_sax_dom_callback_parser if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_object()) { JSON_ASSERT(!key_keep_stack.empty()); + JSON_ASSERT(!key_stack.empty()); const bool placeholder_stored = key_keep_stack.back(); key_keep_stack.pop_back(); + const string_t key = std::move(key_stack.back()); + key_stack.pop_back(); if (placeholder_stored) { - remove_discarded_value(*ref_stack.back()); + remove_discarded_value(*ref_stack.back(), key); } } return {false, nullptr}; @@ -904,8 +965,10 @@ class json_sax_dom_callback_parser JSON_ASSERT(ref_stack.back()->is_object()); // check if we should store an element for the current key JSON_ASSERT(!key_keep_stack.empty()); + JSON_ASSERT(!key_stack.empty()); const bool store_element = key_keep_stack.back(); key_keep_stack.pop_back(); + key_stack.pop_back(); if (!store_element) { @@ -925,6 +988,12 @@ class json_sax_dom_callback_parser std::vector keep_stack {}; // NOLINT(readability-redundant-member-init) /// stack to manage which object keys to keep std::vector key_keep_stack {}; // NOLINT(readability-redundant-member-init) + /// the keys key() stored a placeholder for, in lockstep with key_keep_stack + std::vector key_stack {}; // NOLINT(readability-redundant-member-init) + /// for each open container, the key it is stored under in its parent + /// object, in lockstep with ref_stack; unused where the parent is not an + /// object + std::vector container_key_stack {}; // NOLINT(readability-redundant-member-init) /// helper to hold the reference for the next object element BasicJsonType* object_element = nullptr; /// whether a syntax error occurred diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index c94c477ed..890a7f721 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -11261,6 +11261,11 @@ class json_sax_dom_callback_parser const bool keep = callback(static_cast(ref_stack.size()), parse_event_t::object_start, discarded); keep_stack.push_back(keep); + // the key this object will be stored under, read before handle_value() + // may consume it; kept in lockstep with ref_stack so end_object() can + // find the object in its parent again + container_key_stack.push_back(current_key()); + auto val = handle_value(BasicJsonType::value_t::object, true); ref_stack.push_back(val.second); @@ -11294,6 +11299,9 @@ class json_sax_dom_callback_parser // check callback for the key const bool keep = callback(static_cast(ref_stack.size()), parse_event_t::key, k); key_keep_stack.push_back(keep); + // remember the key so a rejected value can be erased without searching + // the object for it (kept in lockstep with key_keep_stack) + key_stack.push_back(val); // add discarded value at the given key and store the reference for later if (keep && ref_stack.back()) @@ -11335,13 +11343,16 @@ class json_sax_dom_callback_parser JSON_ASSERT(!ref_stack.empty()); JSON_ASSERT(!keep_stack.empty()); + JSON_ASSERT(!container_key_stack.empty()); ref_stack.pop_back(); keep_stack.pop_back(); + const string_t object_key = std::move(container_key_stack.back()); + container_key_stack.pop_back(); if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_structured()) { // remove discarded value - remove_discarded_value(*ref_stack.back()); + remove_discarded_value(*ref_stack.back(), object_key); } return true; @@ -11352,6 +11363,9 @@ class json_sax_dom_callback_parser const bool keep = callback(static_cast(ref_stack.size()), parse_event_t::array_start, discarded); keep_stack.push_back(keep); + // see start_object() + container_key_stack.push_back(current_key()); + auto val = handle_value(BasicJsonType::value_t::array, true); ref_stack.push_back(val.second); @@ -11414,8 +11428,11 @@ class json_sax_dom_callback_parser JSON_ASSERT(!ref_stack.empty()); JSON_ASSERT(!keep_stack.empty()); + JSON_ASSERT(!container_key_stack.empty()); ref_stack.pop_back(); keep_stack.pop_back(); + const string_t object_key = std::move(container_key_stack.back()); + container_key_stack.pop_back(); // remove discarded value if (!ref_stack.empty() && ref_stack.back()) @@ -11429,7 +11446,7 @@ class json_sax_dom_callback_parser // the array is either still stored under its key or was never // stored, leaving the placeholder key() wrote; both show up as // a discarded member of the parent object - remove_discarded_value(*ref_stack.back()); + remove_discarded_value(*ref_stack.back(), object_key); } } @@ -11522,15 +11539,56 @@ class json_sax_dom_callback_parser } #endif - /// remove the discarded value the callback rejected from its parent - static void remove_discarded_value(BasicJsonType& parent) + /*! + @brief the key the value now being handled will be stored under + + Empty unless the enclosing container is an object, in which case it is the + key of the pending key() event. Read before handle_value() consumes that + key, so it is also correct when the value never reaches its parent. + */ + string_t current_key() const { - for (auto it = parent.begin(); it != parent.end(); ++it) + if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_object() + && !key_stack.empty()) { - if (it->is_discarded()) + return key_stack.back(); + } + return string_t{}; + } + + /*! + @brief remove the discarded value the callback rejected from its parent + + A rejected value can only ever be the one most recently added to @a parent: + the last element of an array, or the placeholder key() stored under @a key + in an object. Looking there directly makes this O(1) resp. O(log n), where + searching @a parent for it made a filtering parse quadratic in the number of + members of a single container. + + Finding no discarded value there means none was stored in the first place - + the callback rejected the value before it reached its parent - so there is + nothing to remove. + + @param[in,out] parent the container to remove the rejected value from + @param[in] key the key the value was stored under; unused for arrays + */ + static void remove_discarded_value(BasicJsonType& parent, const string_t& key) + { + if (parent.is_array()) + { + auto& array = *parent.m_data.m_value.array; + if (!array.empty() && array.back().is_discarded()) { - parent.erase(it); - break; + array.pop_back(); + } + } + else if (parent.is_object()) + { + auto& object = *parent.m_data.m_value.object; + const auto it = object.find(key); + if (it != object.end() && it->second.is_discarded()) + { + object.erase(it); } } } @@ -11580,11 +11638,14 @@ class json_sax_dom_callback_parser if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_object()) { JSON_ASSERT(!key_keep_stack.empty()); + JSON_ASSERT(!key_stack.empty()); const bool placeholder_stored = key_keep_stack.back(); key_keep_stack.pop_back(); + const string_t key = std::move(key_stack.back()); + key_stack.pop_back(); if (placeholder_stored) { - remove_discarded_value(*ref_stack.back()); + remove_discarded_value(*ref_stack.back(), key); } } return {false, nullptr}; @@ -11617,8 +11678,10 @@ class json_sax_dom_callback_parser JSON_ASSERT(ref_stack.back()->is_object()); // check if we should store an element for the current key JSON_ASSERT(!key_keep_stack.empty()); + JSON_ASSERT(!key_stack.empty()); const bool store_element = key_keep_stack.back(); key_keep_stack.pop_back(); + key_stack.pop_back(); if (!store_element) { @@ -11638,6 +11701,12 @@ class json_sax_dom_callback_parser std::vector keep_stack {}; // NOLINT(readability-redundant-member-init) /// stack to manage which object keys to keep std::vector key_keep_stack {}; // NOLINT(readability-redundant-member-init) + /// the keys key() stored a placeholder for, in lockstep with key_keep_stack + std::vector key_stack {}; // NOLINT(readability-redundant-member-init) + /// for each open container, the key it is stored under in its parent + /// object, in lockstep with ref_stack; unused where the parent is not an + /// object + std::vector container_key_stack {}; // NOLINT(readability-redundant-member-init) /// helper to hold the reference for the next object element BasicJsonType* object_element = nullptr; /// whether a syntax error occurred diff --git a/tests/src/unit-class_parser.cpp b/tests/src/unit-class_parser.cpp index e0b21d975..34ca5db2b 100644 --- a/tests/src/unit-class_parser.cpp +++ b/tests/src/unit-class_parser.cpp @@ -1768,6 +1768,58 @@ TEST_CASE("parser class") CHECK (j_filtered2 == json({{"foo", {1, 2}}})); } + SECTION("filter many members of one container") + { + // Rejecting a value makes the parser remove the placeholder its key + // event stored. Locating that placeholder used to be a scan of the + // whole parent, which made filtering a large container quadratic: + // 128k members took ~25 s. These cases keep many members alive + // while discarding many others, so the removal cost is the whole + // point; they run in milliseconds when the placeholder is erased + // directly. + constexpr int count = 20000; + + std::string s = "{"; + for (int i = 0; i < count; ++i) + { + // "a" is kept, "z" is discarded + s += "\"a" + std::to_string(i) + "\":" + std::to_string(i) + ","; + s += "\"z" + std::to_string(i) + "\":-1,"; + } + s.back() = '}'; + + const json j_values = json::parse(s, [](int /*unused*/, json::parse_event_t e, const json & parsed) noexcept + { + return !(e == json::parse_event_t::value && parsed == json(-1)); + }); + + CHECK(j_values.size() == count); + CHECK(j_values.at("a0") == json(0)); + CHECK(j_values.at("a" + std::to_string(count - 1)) == json(count - 1)); + CHECK_FALSE(j_values.contains("z0")); + CHECK_FALSE(j_values.contains("z" + std::to_string(count - 1))); + + // the same, but discarding whole containers rather than values, + // which takes the end_object()/end_array() removal path + std::string s_nested = "{"; + for (int i = 0; i < count; ++i) + { + s_nested += "\"a" + std::to_string(i) + "\":" + std::to_string(i) + ","; + s_nested += "\"z" + std::to_string(i) + "\":[1,2],"; + } + s_nested.back() = '}'; + + const json j_arrays = json::parse(s_nested, [](int /*unused*/, json::parse_event_t e, const json& /*unused*/) noexcept + { + return e != json::parse_event_t::array_end; + }); + + CHECK(j_arrays.size() == count); + CHECK(j_arrays.at("a0") == json(0)); + CHECK_FALSE(j_arrays.contains("z0")); + CHECK_FALSE(j_arrays.contains("z" + std::to_string(count - 1))); + } + SECTION("filter specific events") { SECTION("first closing event") From fae364db248a797a24b74c71028e583c62b58ec1 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Fri, 11 Sep 2026 08:22:51 +0200 Subject: [PATCH 10/18] Move the scanned string into the value instead of copying it (#5458) * Move the scanned string into the value instead of copying it The SAX interface documents that the string handed to json_sax::string() may be moved from, and the DOM handlers already move the one handed to binary(). string() did not, so every string value was copy-constructed out of the lexer's token buffer, which then kept the buffer alive at its high-water mark until the next token overwrote it. Moving hands that buffer to the new value instead. The allocation count is unchanged - the value needed one either way - but the copy is gone. jeopardy 247.3 ms -> 240.9 ms (-2.6%) citm_catalog 4.61 ms -> 4.48 ms (-2.8%) 40k 30-char strings 7.80 ms -> 7.64 ms (-2.1%) Note this deliberately does not extend to the object key. Moving the key hands the lexer's buffer - sized for the largest token seen so far - to a key that is usually short, so the next value has to grow a fresh buffer. Measured, that costs 11.9% on a document of many small keys with longer values. Signed-off-by: Niels Lohmann * Spell out the move rationale at every handle_value(std::move) site The comment explaining why the value is moved sat only on json_sax_dom_parser::string(), and the callback parser's string() pointed at it with "see json_sax_dom_parser::string()". That reference cannot be searched for - the function is declared as `bool string(string_t& val)` inside the class, so the qualified name appears nowhere - and the two binary() overloads, which have always moved, carried no explanation at all. Put the same comment on all four sites and name json_sax, which is greppable, instead of a member that is not. Comment-only; no generated code changes. Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann --- include/nlohmann/detail/input/json_sax.hpp | 12 ++++++++++-- single_include/nlohmann/json.hpp | 12 ++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/include/nlohmann/detail/input/json_sax.hpp b/include/nlohmann/detail/input/json_sax.hpp index 684fac047..1627d2326 100644 --- a/include/nlohmann/detail/input/json_sax.hpp +++ b/include/nlohmann/detail/input/json_sax.hpp @@ -222,12 +222,16 @@ class json_sax_dom_parser bool string(string_t& val) { - handle_value(val); + // json_sax documents that the passed value may be moved from, + // so hand the buffer over instead of copying it + handle_value(std::move(val)); return true; } bool binary(binary_t& val) { + // json_sax documents that the passed value may be moved from, + // so hand the buffer over instead of copying it handle_value(std::move(val)); return true; } @@ -532,12 +536,16 @@ class json_sax_dom_callback_parser bool string(string_t& val) { - handle_value(val); + // json_sax documents that the passed value may be moved from, + // so hand the buffer over instead of copying it + handle_value(std::move(val)); return true; } bool binary(binary_t& val) { + // json_sax documents that the passed value may be moved from, + // so hand the buffer over instead of copying it handle_value(std::move(val)); return true; } diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 890a7f721..e2c314c5f 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -10935,12 +10935,16 @@ class json_sax_dom_parser bool string(string_t& val) { - handle_value(val); + // json_sax documents that the passed value may be moved from, + // so hand the buffer over instead of copying it + handle_value(std::move(val)); return true; } bool binary(binary_t& val) { + // json_sax documents that the passed value may be moved from, + // so hand the buffer over instead of copying it handle_value(std::move(val)); return true; } @@ -11245,12 +11249,16 @@ class json_sax_dom_callback_parser bool string(string_t& val) { - handle_value(val); + // json_sax documents that the passed value may be moved from, + // so hand the buffer over instead of copying it + handle_value(std::move(val)); return true; } bool binary(binary_t& val) { + // json_sax documents that the passed value may be moved from, + // so hand the buffer over instead of copying it handle_value(std::move(val)); return true; } From dd50f0eb163f5d2e045d02b4525a62a45084b7ed Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Fri, 11 Sep 2026 08:34:41 +0200 Subject: [PATCH 11/18] Stop CBOR indefinite-length strings from recursing per chunk (#5502) get_cbor_string() and get_cbor_binary() handled the indefinite-length forms (0x7F and 0x5F) by calling themselves once per chunk. Each chunk therefore cost a native stack frame, and since a chunk may itself be an indefinite- length string, an input of repeated 0x7F bytes reached one frame per input byte: 200,000 of them crash the process with SIGSEGV before a single byte is rejected. This is the same defect as #5104, in a path the container-level work does not touch. Count the open levels instead of recursing through them. That is enough here because every chunk is appended to the same result -- get_bytes() writes at result.size() -- so there is no per-level state to keep. The temporary chunk string and its copy into the result go away with the recursion. The definite-length cases move to get_cbor_string_chunk() and get_cbor_binary_chunk() unchanged, including their error messages, which still name 0x7F and 0x5F because those are handled one level up. Behaviour is unchanged. Comparing against develop over the interesting byte sequences -- empty, single-chunk, nested, over-closed and truncated forms, both strings and byte arrays, and an indefinite-length map key -- produces identical values, error codes, messages and byte offsets. The 200,000-level input now reports parse_error.110 at byte 200001 instead of crashing. Note that nesting these is not valid CBOR: RFC 8949, Section 3.2.3 forbids it. This does not change that either way -- it has always been accepted, and rejecting it is a separate decision (#5317, #5325). Should it be rejected later, that is now one condition on the level counter rather than a change to the control flow. Signed-off-by: Niels Lohmann --- .../nlohmann/detail/input/binary_reader.hpp | 188 +++++++++++++----- single_include/nlohmann/json.hpp | 188 +++++++++++++----- tests/src/unit-cbor.cpp | 52 +++++ 3 files changed, 326 insertions(+), 102 deletions(-) diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index 557d7669c..1e05c08f3 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -996,23 +996,21 @@ class binary_reader } /*! - @brief reads a CBOR string + @brief reads a definite-length CBOR string - This function first reads starting bytes to determine the expected - string length and then copies this number of bytes into a string. - Additionally, CBOR's strings with indefinite lengths are supported. + Reads everything @ref get_cbor_string accepts except the indefinite-length + form, which that function handles itself. The bytes are appended to @a + result, so consecutive chunks of an indefinite-length string can be read + into the same string. - @param[out] result created string + @param[out] result string the bytes are appended to @return whether string creation completed - */ - bool get_cbor_string(string_t& result) - { - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, "string"))) - { - return false; - } + @pre @a current is not EOF + */ + bool get_cbor_string_chunk(string_t& result) + { switch (current) { // UTF-8 string (0x00..0x17 bytes follow) @@ -1068,20 +1066,6 @@ class binary_reader return get_number(input_format_t::cbor, len) && get_string(input_format_t::cbor, len, result); } - case 0x7F: // UTF-8 string (indefinite length) - { - while (get() != 0xFF) - { - string_t chunk; - if (!get_cbor_string(chunk)) - { - return false; - } - result.append(chunk); - } - return true; - } - default: { auto last_token = get_token_string(); @@ -1092,23 +1076,82 @@ class binary_reader } /*! - @brief reads a CBOR byte array + @brief reads a CBOR string This function first reads starting bytes to determine the expected - byte array length and then copies this number of bytes into the byte array. - Additionally, CBOR's byte arrays with indefinite lengths are supported. + string length and then copies this number of bytes into a string. + Additionally, CBOR's strings with indefinite lengths are supported. - @param[out] result created byte array + @param[out] result created string + + @return whether string creation completed + */ + bool get_cbor_string(string_t& result) + { + // number of indefinite-length strings that have been opened and not + // closed yet. RFC 8949, Section 3.2.3 does not permit nesting them, + // but this reader has always accepted it, so the open levels are + // counted instead of recursed through, which overflowed the stack for + // an input of repeated 0x7F bytes (see #5104). Every chunk is appended + // to the same result, so no per-level state is needed. + std::size_t open = 0; + + while (true) + { + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, "string"))) + { + return false; + } + + if (current == 0x7F) // UTF-8 string (indefinite length) + { + ++open; + get(); + continue; + } + + // a break marker closes the innermost indefinite-length string; + // outside of one it is not a string and falls through to the error + if (open != 0 && current == 0xFF) + { + if (--open == 0) + { + return true; + } + get(); + continue; + } + + if (JSON_HEDLEY_UNLIKELY(!get_cbor_string_chunk(result))) + { + return false; + } + + if (open == 0) + { + return true; + } + + get(); + } + } + + /*! + @brief reads a definite-length CBOR byte array + + Reads everything @ref get_cbor_binary accepts except the indefinite-length + form, which that function handles itself. The bytes are appended to @a + result, so consecutive chunks of an indefinite-length byte array can be + read into the same byte array. + + @param[out] result byte array the bytes are appended to @return whether byte array creation completed - */ - bool get_cbor_binary(binary_t& result) - { - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, "binary"))) - { - return false; - } + @pre @a current is not EOF + */ + bool get_cbor_binary_chunk(binary_t& result) + { switch (current) { // Binary data (0x00..0x17 bytes follow) @@ -1168,20 +1211,6 @@ class binary_reader get_binary(input_format_t::cbor, len, result); } - case 0x5F: // Binary data (indefinite length) - { - while (get() != 0xFF) - { - binary_t chunk; - if (!get_cbor_binary(chunk)) - { - return false; - } - result.insert(result.end(), chunk.begin(), chunk.end()); - } - return true; - } - default: { auto last_token = get_token_string(); @@ -1191,6 +1220,63 @@ class binary_reader } } + /*! + @brief reads a CBOR byte array + + This function first reads starting bytes to determine the expected + byte array length and then copies this number of bytes into the byte array. + Additionally, CBOR's byte arrays with indefinite lengths are supported. + + @param[out] result created byte array + + @return whether byte array creation completed + */ + bool get_cbor_binary(binary_t& result) + { + // the open indefinite-length byte arrays are counted rather than + // recursed through, for the reason given in @ref get_cbor_string + std::size_t open = 0; + + while (true) + { + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, "binary"))) + { + return false; + } + + if (current == 0x5F) // Binary data (indefinite length) + { + ++open; + get(); + continue; + } + + // a break marker closes the innermost indefinite-length byte + // array; outside of one it falls through to the error below + if (open != 0 && current == 0xFF) + { + if (--open == 0) + { + return true; + } + get(); + continue; + } + + if (JSON_HEDLEY_UNLIKELY(!get_cbor_binary_chunk(result))) + { + return false; + } + + if (open == 0) + { + return true; + } + + get(); + } + } + /*! @brief narrow a definite CBOR array/map length to std::size_t diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index e2c314c5f..44ee77295 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -12945,23 +12945,21 @@ class binary_reader } /*! - @brief reads a CBOR string + @brief reads a definite-length CBOR string - This function first reads starting bytes to determine the expected - string length and then copies this number of bytes into a string. - Additionally, CBOR's strings with indefinite lengths are supported. + Reads everything @ref get_cbor_string accepts except the indefinite-length + form, which that function handles itself. The bytes are appended to @a + result, so consecutive chunks of an indefinite-length string can be read + into the same string. - @param[out] result created string + @param[out] result string the bytes are appended to @return whether string creation completed - */ - bool get_cbor_string(string_t& result) - { - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, "string"))) - { - return false; - } + @pre @a current is not EOF + */ + bool get_cbor_string_chunk(string_t& result) + { switch (current) { // UTF-8 string (0x00..0x17 bytes follow) @@ -13017,20 +13015,6 @@ class binary_reader return get_number(input_format_t::cbor, len) && get_string(input_format_t::cbor, len, result); } - case 0x7F: // UTF-8 string (indefinite length) - { - while (get() != 0xFF) - { - string_t chunk; - if (!get_cbor_string(chunk)) - { - return false; - } - result.append(chunk); - } - return true; - } - default: { auto last_token = get_token_string(); @@ -13041,23 +13025,82 @@ class binary_reader } /*! - @brief reads a CBOR byte array + @brief reads a CBOR string This function first reads starting bytes to determine the expected - byte array length and then copies this number of bytes into the byte array. - Additionally, CBOR's byte arrays with indefinite lengths are supported. + string length and then copies this number of bytes into a string. + Additionally, CBOR's strings with indefinite lengths are supported. - @param[out] result created byte array + @param[out] result created string + + @return whether string creation completed + */ + bool get_cbor_string(string_t& result) + { + // number of indefinite-length strings that have been opened and not + // closed yet. RFC 8949, Section 3.2.3 does not permit nesting them, + // but this reader has always accepted it, so the open levels are + // counted instead of recursed through, which overflowed the stack for + // an input of repeated 0x7F bytes (see #5104). Every chunk is appended + // to the same result, so no per-level state is needed. + std::size_t open = 0; + + while (true) + { + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, "string"))) + { + return false; + } + + if (current == 0x7F) // UTF-8 string (indefinite length) + { + ++open; + get(); + continue; + } + + // a break marker closes the innermost indefinite-length string; + // outside of one it is not a string and falls through to the error + if (open != 0 && current == 0xFF) + { + if (--open == 0) + { + return true; + } + get(); + continue; + } + + if (JSON_HEDLEY_UNLIKELY(!get_cbor_string_chunk(result))) + { + return false; + } + + if (open == 0) + { + return true; + } + + get(); + } + } + + /*! + @brief reads a definite-length CBOR byte array + + Reads everything @ref get_cbor_binary accepts except the indefinite-length + form, which that function handles itself. The bytes are appended to @a + result, so consecutive chunks of an indefinite-length byte array can be + read into the same byte array. + + @param[out] result byte array the bytes are appended to @return whether byte array creation completed - */ - bool get_cbor_binary(binary_t& result) - { - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, "binary"))) - { - return false; - } + @pre @a current is not EOF + */ + bool get_cbor_binary_chunk(binary_t& result) + { switch (current) { // Binary data (0x00..0x17 bytes follow) @@ -13117,20 +13160,6 @@ class binary_reader get_binary(input_format_t::cbor, len, result); } - case 0x5F: // Binary data (indefinite length) - { - while (get() != 0xFF) - { - binary_t chunk; - if (!get_cbor_binary(chunk)) - { - return false; - } - result.insert(result.end(), chunk.begin(), chunk.end()); - } - return true; - } - default: { auto last_token = get_token_string(); @@ -13140,6 +13169,63 @@ class binary_reader } } + /*! + @brief reads a CBOR byte array + + This function first reads starting bytes to determine the expected + byte array length and then copies this number of bytes into the byte array. + Additionally, CBOR's byte arrays with indefinite lengths are supported. + + @param[out] result created byte array + + @return whether byte array creation completed + */ + bool get_cbor_binary(binary_t& result) + { + // the open indefinite-length byte arrays are counted rather than + // recursed through, for the reason given in @ref get_cbor_string + std::size_t open = 0; + + while (true) + { + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, "binary"))) + { + return false; + } + + if (current == 0x5F) // Binary data (indefinite length) + { + ++open; + get(); + continue; + } + + // a break marker closes the innermost indefinite-length byte + // array; outside of one it falls through to the error below + if (open != 0 && current == 0xFF) + { + if (--open == 0) + { + return true; + } + get(); + continue; + } + + if (JSON_HEDLEY_UNLIKELY(!get_cbor_binary_chunk(result))) + { + return false; + } + + if (open == 0) + { + return true; + } + + get(); + } + } + /*! @brief narrow a definite CBOR array/map length to std::size_t diff --git a/tests/src/unit-cbor.cpp b/tests/src/unit-cbor.cpp index 2a6bd41d7..96b30e142 100644 --- a/tests/src/unit-cbor.cpp +++ b/tests/src/unit-cbor.cpp @@ -2035,6 +2035,58 @@ TEST_CASE("CBOR definite length equal to the indefinite-length sentinel") } } +TEST_CASE("CBOR indefinite-length strings do not recurse per chunk") +{ + // Reading an indefinite-length string or byte array used to call itself + // once per chunk, so a payload of repeated 0x7F (or 0x5F) bytes exhausted + // the call stack before any of the input was rejected. The open levels are + // counted now, and the levels below prove the reader still reads the same + // values and reports the same errors at the same byte offsets. + json _; + + SECTION("many open levels are reported, not crashed on") + { + const std::vector input(200000, 0x7F); + CHECK_THROWS_WITH_AS(_ = json::from_cbor(input), "[json.exception.parse_error.110] parse error at byte 200001: syntax error while parsing CBOR string: unexpected end of input", json::parse_error&); + CHECK(json::from_cbor(input, true, false).is_discarded()); + } + + SECTION("many open levels are reported, not crashed on (binary)") + { + const std::vector input(200000, 0x5F); + CHECK_THROWS_WITH_AS(_ = json::from_cbor(input), "[json.exception.parse_error.110] parse error at byte 200001: syntax error while parsing CBOR binary: unexpected end of input", json::parse_error&); + CHECK(json::from_cbor(input, true, false).is_discarded()); + } + + SECTION("chunks are still concatenated") + { + CHECK(json::from_cbor(std::vector({0x7F, 0xFF})) == json("")); + CHECK(json::from_cbor(std::vector({0x7F, 0x61, 0x61, 0xFF})) == json("a")); + // nested indefinite-length strings are concatenated across levels + CHECK(json::from_cbor(std::vector({0x7F, 0x7F, 0x61, 0x61, 0xFF, 0x61, 0x62, 0xFF})) == json("ab")); + CHECK(json::from_cbor(std::vector({0x7F, 0x7F, 0x7F, 0x61, 0x7A, 0xFF, 0xFF, 0xFF})) == json("z")); + CHECK(json::from_cbor(std::vector({0xA1, 0x7F, 0x61, 0x61, 0xFF, 0x01})) == json({{"a", 1}})); + } + + SECTION("chunks are still concatenated (binary)") + { + CHECK(json::from_cbor(std::vector({0x5F, 0x41, 0x61, 0xFF})) == json::binary({0x61})); + CHECK(json::from_cbor(std::vector({0x5F, 0x5F, 0x41, 0x61, 0xFF, 0x41, 0x62, 0xFF})) == json::binary({0x61, 0x62})); + } + + SECTION("a chunk that is not a string is still rejected") + { + CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector({0x7F, 0x7F, 0x00})), "[json.exception.parse_error.113] parse error at byte 3: syntax error while parsing CBOR string: expected length specification (0x60-0x7B) or indefinite string type (0x7F); last byte: 0x00", json::parse_error&); + CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector({0x5F, 0x5F, 0x00})), "[json.exception.parse_error.113] parse error at byte 3: syntax error while parsing CBOR binary: expected length specification (0x40-0x5B) or indefinite binary array type (0x5F); last byte: 0x00", json::parse_error&); + } + + SECTION("a break marker outside an indefinite-length string is not a string") + { + // 0xFF only closes a string that was opened; on its own it is not one + CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector({0xA1, 0xFF, 0x01})), "[json.exception.parse_error.113] parse error at byte 2: syntax error while parsing CBOR string: expected length specification (0x60-0x7B) or indefinite string type (0x7F); last byte: 0xFF", json::parse_error&); + } +} + TEST_CASE("CBOR roundtrips" * doctest::skip()) { SECTION("input from flynn") From 4bb2de8cc5c0bf7b7d1664540c36c12aa2e8fce8 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Fri, 11 Sep 2026 08:34:41 +0200 Subject: [PATCH 12/18] Reject a nested BJData ndarray dimension vector where it is read (#5503) get_ubjson_size_type() takes an inside_ndarray parameter saying whether it is being called for an ndarray's dimension vector, where another ndarray is not allowed. It then seeded the flag it passes down to get_ubjson_size_value() with `false` rather than with that parameter, and only consulted inside_ndarray afterwards, on the '$' branch. So on the '#' branch nothing stopped the descent: every "#[" pair of an input like "[" followed by "#[#[#[..." opened another dimension vector, several native stack frames deeper each time, and the recursion was only reported on the way back out. 100,000 pairs crash the process. This is #5104 again, in a path that has nothing to do with containers. Seed the flag with inside_ndarray, which is what get_ubjson_size_value() documents it wants: "for input, `true` means already inside an ndarray vector or ndarray dimension is not allowed". The nested '[' is then refused where it is read, so the length of the chain no longer matters. Both post-checks gain `&& !inside_ndarray`, because an ndarray was found *here* only if the flag flipped -- get_ubjson_size_value() only ever returns `true` when its initial value was `false`, as its documentation says. With that, the "ndarray can not be recursive" branch is unreachable: a recursive ndarray is now caught one level earlier, and reported as "ndarray dimensional vector is not allowed" like every other nested dimension vector. Three existing expectations move accordingly (vR2, vR4, vR6). All three now fail earlier, and all three now report the same error that vR1, vR5 and vH already reported for the same shape, which is the more consistent outcome. Everything else is unchanged: valid 1D and 2D ndarrays, optimized containers and plain arrays produce identical results, and unit-ubjson is untouched. Signed-off-by: Niels Lohmann --- .../nlohmann/detail/input/binary_reader.hpp | 19 ++++++++++------- single_include/nlohmann/json.hpp | 19 ++++++++++------- tests/src/unit-bjdata.cpp | 21 ++++++++++++++++--- 3 files changed, 40 insertions(+), 19 deletions(-) diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index 1e05c08f3..f61aaf32f 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -2477,7 +2477,12 @@ class binary_reader { result.first = npos; // size result.second = 0; // type - bool is_ndarray = false; + // seed the flag with the caller's context: inside an ndarray dimension + // vector another ndarray is not allowed, and get_ubjson_size_value() + // rejects it up front instead of reading it and reporting afterwards. + // Seeding it with `false` made every '#' of a "[#[#[..." chain descend + // another level, which overflowed the stack (see #5104). + bool is_ndarray = inside_ndarray; get_ignore_noop(); @@ -2510,13 +2515,11 @@ class binary_reader } const bool is_error = get_ubjson_size_value(result.first, is_ndarray); - if (input_format == input_format_t::bjdata && is_ndarray) + // an ndarray was read here only if the flag flipped; when it was + // seeded true, get_ubjson_size_value() already rejected the nested + // dimension vector + if (input_format == input_format_t::bjdata && is_ndarray && !inside_ndarray) { - if (inside_ndarray) - { - return sax->parse_error(chars_read, get_token_string(), parse_error::create(112, chars_read, - exception_message(input_format, "ndarray can not be recursive", "size"), nullptr)); - } result.second |= (1 << 8); // use bit 8 to indicate ndarray, all UBJSON and BJData markers should be ASCII letters } return is_error; @@ -2525,7 +2528,7 @@ class binary_reader if (current == '#') { const bool is_error = get_ubjson_size_value(result.first, is_ndarray); - if (input_format == input_format_t::bjdata && is_ndarray) + if (input_format == input_format_t::bjdata && is_ndarray && !inside_ndarray) { return sax->parse_error(chars_read, get_token_string(), parse_error::create(112, chars_read, exception_message(input_format, "ndarray requires both type and size", "size"), nullptr)); diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 44ee77295..b155caa63 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -14426,7 +14426,12 @@ class binary_reader { result.first = npos; // size result.second = 0; // type - bool is_ndarray = false; + // seed the flag with the caller's context: inside an ndarray dimension + // vector another ndarray is not allowed, and get_ubjson_size_value() + // rejects it up front instead of reading it and reporting afterwards. + // Seeding it with `false` made every '#' of a "[#[#[..." chain descend + // another level, which overflowed the stack (see #5104). + bool is_ndarray = inside_ndarray; get_ignore_noop(); @@ -14459,13 +14464,11 @@ class binary_reader } const bool is_error = get_ubjson_size_value(result.first, is_ndarray); - if (input_format == input_format_t::bjdata && is_ndarray) + // an ndarray was read here only if the flag flipped; when it was + // seeded true, get_ubjson_size_value() already rejected the nested + // dimension vector + if (input_format == input_format_t::bjdata && is_ndarray && !inside_ndarray) { - if (inside_ndarray) - { - return sax->parse_error(chars_read, get_token_string(), parse_error::create(112, chars_read, - exception_message(input_format, "ndarray can not be recursive", "size"), nullptr)); - } result.second |= (1 << 8); // use bit 8 to indicate ndarray, all UBJSON and BJData markers should be ASCII letters } return is_error; @@ -14474,7 +14477,7 @@ class binary_reader if (current == '#') { const bool is_error = get_ubjson_size_value(result.first, is_ndarray); - if (input_format == input_format_t::bjdata && is_ndarray) + if (input_format == input_format_t::bjdata && is_ndarray && !inside_ndarray) { return sax->parse_error(chars_read, get_token_string(), parse_error::create(112, chars_read, exception_message(input_format, "ndarray requires both type and size", "size"), nullptr)); diff --git a/tests/src/unit-bjdata.cpp b/tests/src/unit-bjdata.cpp index 78ddf5d8c..3130bb720 100644 --- a/tests/src/unit-bjdata.cpp +++ b/tests/src/unit-bjdata.cpp @@ -3335,8 +3335,10 @@ TEST_CASE("BJData") CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vR1), "[json.exception.parse_error.113] parse error at byte 6: syntax error while parsing BJData size: ndarray dimensional vector is not allowed", json::parse_error&); CHECK(json::from_bjdata(vR1, true, false).is_discarded()); + // a dimension vector that opens another one is rejected where the + // nested '[' is read, rather than after it has been descended into std::vector const vR2 = {'[', '$', 'i', '#', '[', '#', '[', 'i', 1, ']', ']', 1}; - CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vR2), "[json.exception.parse_error.113] parse error at byte 11: syntax error while parsing BJData size: expected length type specification (U, i, u, I, m, l, M, L) after '#'; last byte: 0x5D", json::parse_error&); + CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vR2), "[json.exception.parse_error.113] parse error at byte 7: syntax error while parsing BJData size: ndarray dimensional vector is not allowed", json::parse_error&); CHECK(json::from_bjdata(vR2, true, false).is_discarded()); std::vector const vR3 = {'[', '#', '[', 'i', '2', 'i', 2, ']'}; @@ -3344,7 +3346,7 @@ TEST_CASE("BJData") CHECK(json::from_bjdata(vR3, true, false).is_discarded()); std::vector const vR4 = {'[', '$', 'i', '#', '[', '$', 'i', '#', '[', 'i', 1, ']', 1}; - CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vR4), "[json.exception.parse_error.110] parse error at byte 14: syntax error while parsing BJData number: unexpected end of input", json::parse_error&); + CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vR4), "[json.exception.parse_error.113] parse error at byte 9: syntax error while parsing BJData size: ndarray dimensional vector is not allowed", json::parse_error&); CHECK(json::from_bjdata(vR4, true, false).is_discarded()); std::vector const vR5 = {'[', '$', 'i', '#', '[', '[', '[', ']', ']', ']'}; @@ -3352,12 +3354,25 @@ TEST_CASE("BJData") CHECK(json::from_bjdata(vR5, true, false).is_discarded()); std::vector const vR6 = {'[', '$', 'i', '#', '[', '$', 'i', '#', '[', 'i', '2', 'i', 2, ']'}; - CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vR6), "[json.exception.parse_error.112] parse error at byte 14: syntax error while parsing BJData size: ndarray can not be recursive", json::parse_error&); + CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vR6), "[json.exception.parse_error.113] parse error at byte 9: syntax error while parsing BJData size: ndarray dimensional vector is not allowed", json::parse_error&); CHECK(json::from_bjdata(vR6, true, false).is_discarded()); std::vector const vH = {'[', 'H', '[', '#', '[', '$', 'i', '#', '[', 'i', '2', 'i', 2, ']'}; CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vH), "[json.exception.parse_error.113] parse error at byte 3: syntax error while parsing BJData size: ndarray dimensional vector is not allowed", json::parse_error&); CHECK(json::from_bjdata(vH, true, false).is_discarded()); + + // Every "#[" of this chain used to open another dimension vector + // and cost several stack frames before anything was rejected, so a + // long enough chain crashed the process (see #5104). The nested + // vector is refused where it is read, so the length is irrelevant. + std::vector vRdeep = {'['}; + for (std::size_t i = 0; i < 100000; ++i) + { + vRdeep.push_back('#'); + vRdeep.push_back('['); + } + CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vRdeep), "[json.exception.parse_error.113] parse error at byte 5: syntax error while parsing BJData size: ndarray dimensional vector is not allowed", json::parse_error&); + CHECK(json::from_bjdata(vRdeep, true, false).is_discarded()); } SECTION("objects") From 44f8ec30e93dd70d3d8f8ef8647b5fab2dff0481 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Fri, 11 Sep 2026 08:34:42 +0200 Subject: [PATCH 13/18] Bound UBJSON optimized arrays of a valueless type (#5504) * Bound UBJSON optimized arrays of a valueless type An element of type 'Z' (null), 'T' (true) or 'F' (false) is encoded by its type marker alone, so an optimized UBJSON array of one of those has no payload: reading an element consumes no input at all. Its declared count is therefore the only thing that decides how much is allocated, and nothing bounded it. "[$Z#l" and a four-byte count is nine bytes of input describing two billion values; #2793 reports 35 GB and 150 seconds from ten bytes, and OSS-Fuzz has an out-of-memory and a timeout report for the same shape. Every other type costs at least one byte per element, so the end of the input bounds it. 'N' (no-op) is already skipped rather than stored. Objects are not affected either: each element is preceded by its key, which costs bytes. And BJData already refuses these markers as an optimized type, so this is a plain UBJSON matter. Reject a count above 1,048,576 elements for those three types with out_of_range.408, the code this reader already uses for a declared size it will not honour. The check runs before the SAX start event, so no container is opened and then abandoned. Rejecting on the read side alone would break the guarantee that anything to_ubjson() writes can be read back, and would trip the round-trip assertion in fuzzer-parse_ubjson.cpp. So the writer falls back to the unoptimized encoding, one byte per element, for arrays of these types above the same limit. Its decision depends only on the array's size, which is identical for a value and for anything parsed back from it, so the round trip is stable. No existing test changes: the largest such count in the test suite is 65,793. The excessive-size test that already used this shape still passes, now rejected a little earlier than by the max_size() check it used to reach. Signed-off-by: Niels Lohmann * Note the 1,048,576 valueless-array limit as (1 << 20) in the docs Addresses review feedback from @gregmarr on PR #5504: spell out the binary/hex form next to the decimal count so it reads as the round power-of-two it is, matching how include/nlohmann/detail/input/binary_reader.hpp defines max_valueless_container_size. Applied in both docs/exceptions.md and ubjson.md, as requested. Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann --- .../docs/features/binary_formats/ubjson.md | 7 +++ docs/mkdocs/docs/home/exceptions.md | 9 +++ .../nlohmann/detail/input/binary_reader.hpp | 31 ++++++++++ .../nlohmann/detail/output/binary_writer.hpp | 12 +++- single_include/nlohmann/json.hpp | 43 ++++++++++++- tests/src/unit-ubjson.cpp | 61 +++++++++++++++++++ 6 files changed, 161 insertions(+), 2 deletions(-) diff --git a/docs/mkdocs/docs/features/binary_formats/ubjson.md b/docs/mkdocs/docs/features/binary_formats/ubjson.md index 76956d60a..be545b9fe 100644 --- a/docs/mkdocs/docs/features/binary_formats/ubjson.md +++ b/docs/mkdocs/docs/features/binary_formats/ubjson.md @@ -69,6 +69,13 @@ The library uses the following mapping from JSON values types to UBJSON types ac Note that `use_size = true` alone may result in larger representations - the benefit of this parameter is that the receiving side is immediately informed on the number of elements of the container. + An array whose type marker is `Z` (null), `T` (true) or `F` (false) stores no payload at all, because the marker + already is the value. Its declared count is therefore the only thing that decides how much memory the receiving side + allocates, and a handful of bytes can describe billions of elements. `from_ubjson` rejects such an array with + [`out_of_range.408`](../../home/exceptions.md#jsonexceptionout_of_range408) when the count exceeds 1,048,576 + (`1 << 20`), and `to_ubjson` writes longer arrays of these types without the annotation, so any value it produces + can be read back. + !!! info "Binary values" If the JSON data contains the binary type, the value stored is a list of integers, as suggested by the UBJSON diff --git a/docs/mkdocs/docs/home/exceptions.md b/docs/mkdocs/docs/home/exceptions.md index 8c6649ef2..09cc8e178 100644 --- a/docs/mkdocs/docs/home/exceptions.md +++ b/docs/mkdocs/docs/home/exceptions.md @@ -868,6 +868,12 @@ The size of an array or object in a [binary format](../features/binary_formats/i the size following `#` for [UBJSON](../features/binary_formats/ubjson.md)/[BJData](../features/binary_formats/bjdata.md), or the encoded length for [CBOR](../features/binary_formats/cbor.md). +The exception is also thrown for a [UBJSON](../features/binary_formats/ubjson.md) array of a type that is encoded by its +marker alone (`Z`, `T` or `F`) whose declared count exceeds 1,048,576 (`1 << 20`). Such an array has no payload, so its +count alone decides how much memory is allocated, and a handful of bytes would otherwise describe billions of values. +[`to_ubjson`](../api/basic_json/to_ubjson.md) writes longer arrays of these types without the size and type annotation, +so any value it produces can still be read back. + !!! failure "Example messages" ``` @@ -879,6 +885,9 @@ or the encoded length for [CBOR](../features/binary_formats/cbor.md). ``` [json.exception.out_of_range.408] syntax error while parsing CBOR size: excessive map size ``` + ``` + [json.exception.out_of_range.408] syntax error while parsing UBJSON size: excessive array size + ``` ### json.exception.out_of_range.409 diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index f61aaf32f..f699de1b9 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -58,6 +58,26 @@ inline bool little_endianness(int num = 1) noexcept return *reinterpret_cast(&num) == 1; } +/*! +@brief largest element count accepted for a UBJSON container of a valueless type + +An element of type 'Z' (null), 'T' (true) or 'F' (false) is encoded by its +type marker alone, so an optimized container of one of those types has no +payload at all and its declared count is the only thing that decides how much +is allocated: `[$Z#L` followed by a large count turns some ten bytes of input +into that many values (see #2793, which reports 35 GB and 150 seconds). Every +other type costs at least one byte per element and is bounded by the end of +the input. + +This is a sanity bound rather than a security boundary, and it is far above +any container met in practice. @ref binary_writer falls back to the +unoptimized encoding for longer containers, so that a value serialized by +this library can always be read back. + +@sa https://github.com/nlohmann/json/issues/2793 +*/ +JSON_INLINE_VARIABLE constexpr std::size_t max_valueless_container_size = 1 << 20; + /////////////////// // binary reader // /////////////////// @@ -2799,6 +2819,17 @@ class binary_reader if (size_and_type.first != npos) { + // reading an element of a valueless type consumes no input, so the + // declared count alone decides how much is allocated; the check is + // made before the start event so that no container is opened that + // is then abandoned. See @ref max_valueless_container_size. + if (JSON_HEDLEY_UNLIKELY((size_and_type.second == 'Z' || size_and_type.second == 'T' || size_and_type.second == 'F') + && size_and_type.first > max_valueless_container_size)) + { + return sax->parse_error(chars_read, get_token_string(), out_of_range::create(408, + exception_message(input_format, "excessive array size", "size"), nullptr)); + } + if (JSON_HEDLEY_UNLIKELY(!sax->start_array(size_and_type.first))) { return false; diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 9514f0fcd..28290de3e 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -826,7 +826,17 @@ class binary_writer std::vector bjdx = {'[', '{', 'S', 'H', 'T', 'F', 'N', 'Z'}; // excluded markers in bjdata optimized type - if (same_prefix && !(use_bjdata && std::find(bjdx.begin(), bjdx.end(), first_prefix) != bjdx.end())) + // an optimized array of a valueless type carries no payload, so a + // reader has nothing but the declared count to bound the allocation + // by and refuses an excessive one. Write the unoptimized form for + // those, at one byte per element, so the result can be read back. + // Objects are not affected: every element is preceded by its key. + const bool valueless_type = (first_prefix == 'Z' || first_prefix == 'T' || first_prefix == 'F'); + const bool excessive_valueless = valueless_type + && j.m_data.m_value.array->size() > detail::max_valueless_container_size; + + if (same_prefix && !excessive_valueless + && !(use_bjdata && std::find(bjdx.begin(), bjdx.end(), first_prefix) != bjdx.end())) { prefix_required = false; oa->write_character(to_char_type('$')); diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index b155caa63..e8899af56 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -12007,6 +12007,26 @@ inline bool little_endianness(int num = 1) noexcept return *reinterpret_cast(&num) == 1; } +/*! +@brief largest element count accepted for a UBJSON container of a valueless type + +An element of type 'Z' (null), 'T' (true) or 'F' (false) is encoded by its +type marker alone, so an optimized container of one of those types has no +payload at all and its declared count is the only thing that decides how much +is allocated: `[$Z#L` followed by a large count turns some ten bytes of input +into that many values (see #2793, which reports 35 GB and 150 seconds). Every +other type costs at least one byte per element and is bounded by the end of +the input. + +This is a sanity bound rather than a security boundary, and it is far above +any container met in practice. @ref binary_writer falls back to the +unoptimized encoding for longer containers, so that a value serialized by +this library can always be read back. + +@sa https://github.com/nlohmann/json/issues/2793 +*/ +JSON_INLINE_VARIABLE constexpr std::size_t max_valueless_container_size = 1 << 20; + /////////////////// // binary reader // /////////////////// @@ -14748,6 +14768,17 @@ class binary_reader if (size_and_type.first != npos) { + // reading an element of a valueless type consumes no input, so the + // declared count alone decides how much is allocated; the check is + // made before the start event so that no container is opened that + // is then abandoned. See @ref max_valueless_container_size. + if (JSON_HEDLEY_UNLIKELY((size_and_type.second == 'Z' || size_and_type.second == 'T' || size_and_type.second == 'F') + && size_and_type.first > max_valueless_container_size)) + { + return sax->parse_error(chars_read, get_token_string(), out_of_range::create(408, + exception_message(input_format, "excessive array size", "size"), nullptr)); + } + if (JSON_HEDLEY_UNLIKELY(!sax->start_array(size_and_type.first))) { return false; @@ -19200,7 +19231,17 @@ class binary_writer std::vector bjdx = {'[', '{', 'S', 'H', 'T', 'F', 'N', 'Z'}; // excluded markers in bjdata optimized type - if (same_prefix && !(use_bjdata && std::find(bjdx.begin(), bjdx.end(), first_prefix) != bjdx.end())) + // an optimized array of a valueless type carries no payload, so a + // reader has nothing but the declared count to bound the allocation + // by and refuses an excessive one. Write the unoptimized form for + // those, at one byte per element, so the result can be read back. + // Objects are not affected: every element is preceded by its key. + const bool valueless_type = (first_prefix == 'Z' || first_prefix == 'T' || first_prefix == 'F'); + const bool excessive_valueless = valueless_type + && j.m_data.m_value.array->size() > detail::max_valueless_container_size; + + if (same_prefix && !excessive_valueless + && !(use_bjdata && std::find(bjdx.begin(), bjdx.end(), first_prefix) != bjdx.end())) { prefix_required = false; oa->write_character(to_char_type('$')); diff --git a/tests/src/unit-ubjson.cpp b/tests/src/unit-ubjson.cpp index b59ac6b38..03446878f 100644 --- a/tests/src/unit-ubjson.cpp +++ b/tests/src/unit-ubjson.cpp @@ -2149,6 +2149,67 @@ TEST_CASE("UBJSON") } } +TEST_CASE("UBJSON optimized arrays of a valueless type are bounded") +{ + // An element of type 'Z', 'T' or 'F' is encoded by its marker alone, so an + // optimized array of one of those has no payload and the declared count is + // the only thing deciding how much is allocated. Ten bytes used to produce + // billions of values (#2793); every other type costs at least one byte per + // element and is bounded by the end of the input. + json _; + + SECTION("an excessive count is rejected") + { + // 'l' is a big-endian int32: 0x7FFFFFFF elements, about 34 GB of value + for (const auto marker : + {'Z', 'T', 'F' + }) + { + const std::vector input = {'[', '$', static_cast(marker), '#', 'l', 0x7F, 0xFF, 0xFF, 0xFF}; + CHECK_THROWS_WITH_AS(_ = json::from_ubjson(input), "[json.exception.out_of_range.408] syntax error while parsing UBJSON size: excessive array size", json::out_of_range&); + CHECK(json::from_ubjson(input, true, false).is_discarded()); + } + } + + SECTION("ordinary counts are unaffected") + { + CHECK(json::from_ubjson(std::vector({'[', '$', 'Z', '#', 'i', 3})) == json({nullptr, nullptr, nullptr})); + CHECK(json::from_ubjson(std::vector({'[', '$', 'T', '#', 'i', 2})) == json({true, true})); + CHECK(json::from_ubjson(std::vector({'[', '$', 'F', '#', 'i', 2})) == json({false, false})); + // 'N' is a no-op rather than a value, and still yields an empty array + CHECK(json::from_ubjson(std::vector({'[', '$', 'N', '#', 'i', 2})) == json::array()); + } + + SECTION("a type with a payload is unaffected") + { + // A count past the limit is not rejected for 'U', which costs a byte + // per element and is bounded by the end of the input instead. The + // count is kept just past the limit rather than made huge, because a + // count that also exceeds the array's max_size() is reported as + // out_of_range before the input runs out, and max_size() depends on + // the width of std::size_t. + const std::vector input = {'[', '$', 'U', '#', 'l', 0x00, 0x10, 0x00, 0x01}; + CHECK_THROWS_WITH_AS(_ = json::from_ubjson(input), "[json.exception.parse_error.110] parse error at byte 10: syntax error while parsing UBJSON number: unexpected end of input", json::parse_error&); + CHECK(json::from_ubjson(input, true, false).is_discarded()); + } + + SECTION("the writer stays within what the reader accepts") + { + // below the limit the optimized form is used and is tiny; above it the + // writer falls back so that the result can still be read back + json const at_limit(1048576, nullptr); + const auto v_at_limit = json::to_ubjson(at_limit, true, true); + CHECK(v_at_limit.size() == 9); + CHECK(v_at_limit.at(1) == '$'); + CHECK(json::from_ubjson(v_at_limit) == at_limit); + + json const above_limit(1048577, nullptr); + const auto v_above_limit = json::to_ubjson(above_limit, true, true); + CHECK(v_above_limit.at(1) != '$'); + CHECK(json::from_ubjson(v_above_limit) == above_limit); + } +} + TEST_CASE("Universal Binary JSON Specification Examples 1") { SECTION("Null Value") From d9c55eb225219bf0402e5529c81ab314ac253a6e Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Fri, 11 Sep 2026 08:34:42 +0200 Subject: [PATCH 14/18] Split unit-regression2.cpp so the MinGW linker can relocate it (#5511) Linking test-regression2 with clang and MinGW fails with relocation truncated to fit: IMAGE_REL_AMD64_REL32 against `.rdata' once the translation unit grows past a certain size: the code can no longer reach the read-only data it references within the range of a 32-bit relocation. The file is one of the largest in the test suite and had been sitting just under that limit, so an unrelated change elsewhere in the library is enough to tip it over. It is already the second such file -- unit-regression1.cpp was split for size before -- and windows.yml already carries a workaround for the same limit hitting the debug sections of this same target, where -g0 was enough because that relocation was against `.debug_line'. This one is against `.rdata', which no compiler flag avoids. Move the second half of the regression tests, and the helper types only they use, into unit-regression3.cpp. The sections are independent -- every statement in "regression tests 2" was already inside a SECTION -- so they move unchanged, and the counts confirm nothing was lost: 168 assertions before the split, 50 plus 118 after. The result is that both files are comfortably smaller than the one that used to link, measured with clang at -O1 for C++20: read-only data text object before 58,233 1,287,764 3,158,120 unit-regression2.cpp 48,161 1,012,988 2,522,296 unit-regression3.cpp 41,710 772,704 1,878,880 No CMake change is needed: tests/CMakeLists.txt globs src/unit-*.cpp, so the new file is picked up and built for every standard like its siblings. CONTRIBUTING.md pointed contributors at unit-regression2.cpp for new bug tests; it now points at the smaller file and says why the two exist, so the split does not quietly undo itself. Signed-off-by: Niels Lohmann --- .github/CONTRIBUTING.md | 4 +- tests/src/unit-regression2.cpp | 808 ----------------------------- tests/src/unit-regression3.cpp | 899 +++++++++++++++++++++++++++++++++ 3 files changed, 902 insertions(+), 809 deletions(-) create mode 100644 tests/src/unit-regression3.cpp diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index b0f65230c..68f82c474 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -108,7 +108,9 @@ The tests are located in [`tests/src/unit-*.cpp`](https://github.com/nlohmann/js are structured along the features of the library or the nature of the tests. Usually, it should be clear from the context which existing file needs to be extended, and only very few cases require creating new test files. -When fixing a bug, edit `unit-regression2.cpp` and add a section referencing the fixed issue. +When fixing a bug, edit `unit-regression3.cpp` and add a section referencing the fixed issue. +`unit-regression2.cpp` holds the older tests; the two files exist because a single one grew large enough for the +MinGW linker to fail relocating it, so please keep adding to the smaller file rather than growing the larger one. #### Exceptions diff --git a/tests/src/unit-regression2.cpp b/tests/src/unit-regression2.cpp index 6c30e3503..4280ec361 100644 --- a/tests/src/unit-regression2.cpp +++ b/tests/src/unit-regression2.cpp @@ -241,209 +241,6 @@ class my_allocator : public std::allocator }; }; -///////////////////////////////////////////////////////////////////// -// for #3077 -///////////////////////////////////////////////////////////////////// - -class FooAlloc -{}; - -class Foo -{ - public: - explicit Foo(const FooAlloc& /* unused */ = FooAlloc()) {} - - bool value = false; -}; - -class FooBar -{ - public: - Foo foo{}; // NOLINT(readability-redundant-member-init) -}; - -inline void from_json(const nlohmann::json& j, FooBar& fb) // NOLINT(misc-use-internal-linkage) -{ - j.at("value").get_to(fb.foo.value); -} - -///////////////////////////////////////////////////////////////////// -// for #3171 -///////////////////////////////////////////////////////////////////// - -struct for_3171_base // NOLINT(cppcoreguidelines-special-member-functions) -{ - for_3171_base(const std::string& /*unused*/ = {}) {} - virtual ~for_3171_base(); - - for_3171_base(const for_3171_base& other) // NOLINT(hicpp-use-equals-default,modernize-use-equals-default) - : str(other.str) - {} - - for_3171_base& operator=(const for_3171_base& other) - { - if (this != &other) - { - str = other.str; - } - return *this; - } - - for_3171_base(for_3171_base&& other) noexcept - : str(std::move(other.str)) - {} - - for_3171_base& operator=(for_3171_base&& other) noexcept - { - if (this != &other) - { - str = std::move(other.str); - } - return *this; - } - - virtual void _from_json(const json& j) - { - j.at("str").get_to(str); - } - - std::string str{}; // NOLINT(readability-redundant-member-init) -}; - -for_3171_base::~for_3171_base() = default; - -struct for_3171_derived : public for_3171_base -{ - for_3171_derived() = default; - ~for_3171_derived() override; - explicit for_3171_derived(const std::string& /*unused*/) { } - - for_3171_derived(const for_3171_derived& other) // NOLINT(hicpp-use-equals-default,modernize-use-equals-default) - : for_3171_base(other) - {} - - for_3171_derived& operator=(const for_3171_derived& other) - { - if (this != &other) - { - for_3171_base::operator=(other); // Call base class assignment operator - } - return *this; - } - - for_3171_derived(for_3171_derived&& other) noexcept - : for_3171_base(std::move(other)) - {} - - for_3171_derived& operator=(for_3171_derived&& other) noexcept - { - if (this != &other) - { - for_3171_base::operator=(std::move(other)); // Call base class move assignment operator - } - return *this; - } -}; - -for_3171_derived::~for_3171_derived() = default; - -inline void from_json(const json& j, for_3171_base& tb) // NOLINT(misc-use-internal-linkage) -{ - tb._from_json(j); -} - -///////////////////////////////////////////////////////////////////// -// for #3312 -///////////////////////////////////////////////////////////////////// - -#ifdef JSON_HAS_CPP_20 -struct for_3312 -{ - std::string name; -}; - -inline void from_json(const json& j, for_3312& obj) // NOLINT(misc-use-internal-linkage) -{ - j.at("name").get_to(obj.name); -} -#endif - -///////////////////////////////////////////////////////////////////// -// for #3204 -///////////////////////////////////////////////////////////////////// - -struct for_3204_foo -{ - for_3204_foo() = default; - explicit for_3204_foo(std::string /*unused*/) {} // NOLINT(performance-unnecessary-value-param) -}; - -struct for_3204_bar -{ - enum constructed_from_t // NOLINT(cppcoreguidelines-use-enum-class) - { - constructed_from_none = 0, - constructed_from_foo = 1, - constructed_from_json = 2 - }; - - explicit for_3204_bar(std::function /*unused*/) noexcept // NOLINT(performance-unnecessary-value-param) - : constructed_from(constructed_from_foo) {} - explicit for_3204_bar(std::function /*unused*/) noexcept // NOLINT(performance-unnecessary-value-param) - : constructed_from(constructed_from_json) {} - - constructed_from_t constructed_from = constructed_from_none; -}; - -///////////////////////////////////////////////////////////////////// -// for #3333 -///////////////////////////////////////////////////////////////////// - -struct for_3333 final -{ - for_3333(int x_ = 0, int y_ = 0) : x(x_), y(y_) {} - - template - for_3333(const T& /*unused*/) - { - CHECK(false); - } - - int x = 0; - int y = 0; -}; - -template <> -inline for_3333::for_3333(const json& j) - : for_3333(j.value("x", 0), j.value("y", 0)) -{} - -///////////////////////////////////////////////////////////////////// -// for #3810 -///////////////////////////////////////////////////////////////////// - -struct Example_3810 -{ - int bla{}; - - Example_3810() = default; -}; - -NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Example_3810, bla) // NOLINT(misc-use-internal-linkage) - -///////////////////////////////////////////////////////////////////// -// for #4740 -///////////////////////////////////////////////////////////////////// - -#ifdef JSON_HAS_CPP_17 -struct Example_4740 -{ - std::optional host = std::nullopt; - std::optional port = std::nullopt; - NLOHMANN_DEFINE_TYPE_INTRUSIVE_WITH_DEFAULT(Example_4740, host, port) -}; -#endif - TEST_CASE("regression tests 2") { SECTION("issue #1001 - Fix memory leak during parser callback") @@ -964,611 +761,6 @@ TEST_CASE("regression tests 2") CHECK(j == k); } -#if JSON_HAS_FILESYSTEM || JSON_HAS_EXPERIMENTAL_FILESYSTEM - // JSON_HAS_CPP_17 (do not remove; see note at top of file) - SECTION("issue #3070 - Version 3.10.3 breaks backward-compatibility with 3.10.2 ") - { - nlohmann::detail::std_fs::path text_path("/tmp/text.txt"); - const json j(text_path); - - const auto j_path = j.get(); - CHECK(j_path == text_path); - -#if DOCTEST_CLANG || DOCTEST_GCC >= DOCTEST_COMPILER(8, 4, 0) - // only known to work on Clang and GCC >=8.4 - CHECK_THROWS_WITH_AS(nlohmann::detail::std_fs::path(json(1)), "[json.exception.type_error.302] type must be string, but is number", json::type_error); -#endif - } -#endif - - SECTION("issue #3077 - explicit constructor with default does not compile") - { - json j; - j[0]["value"] = true; - std::vector foo; - j.get_to(foo); - } - - SECTION("issue #3108 - ordered_json doesn't support range based erase") - { - ordered_json j = {1, 2, 2, 4}; - - auto last = std::unique(j.begin(), j.end()); - j.erase(last, j.end()); - - CHECK(j.dump() == "[1,2,4]"); - - j.erase(std::remove_if(j.begin(), j.end(), [](const ordered_json & val) - { - return val == 2; - }), j.end()); - - CHECK(j.dump() == "[1,4]"); - } - - SECTION("issue #3343 - json and ordered_json are not interchangeable") - { - json::object_t jobj({ { "product", "one" } }); - ordered_json::object_t ojobj({{"product", "one"}}); - - auto jit = jobj.begin(); - auto ojit = ojobj.begin(); - - CHECK(jit->first == ojit->first); - CHECK(jit->second.get() == ojit->second.get()); - } - - SECTION("issue #3171 - if class is_constructible from std::string wrong from_json overload is being selected, compilation failed") - { - const json j{{ "str", "value"}}; - - // failed with: error: no match for ‘operator=’ (operand types are ‘for_3171_derived’ and ‘const nlohmann::basic_json<>::string_t’ - // {aka ‘const std::__cxx11::basic_string’}) - // s = *j.template get_ptr(); - auto td = j.get(); - - CHECK(td.str == "value"); - } - -#ifdef JSON_HAS_CPP_20 - SECTION("issue #3312 - Parse to custom class from unordered_json breaks on G++11.2.0 with C++20") - { - // see test for #3171 - const ordered_json j = {{"name", "class"}}; - for_3312 obj{}; - - j.get_to(obj); - - CHECK(obj.name == "class"); - } -#endif - -#if defined(JSON_HAS_CPP_17) && JSON_USE_IMPLICIT_CONVERSIONS - SECTION("issue #3428 - Error occurred when converting nlohmann::json to std::any") - { - const json j; - const std::any a1 = j; - std::any&& a2 = j; - - CHECK(a1.type() == typeid(j)); - CHECK(a2.type() == typeid(j)); - } -#endif - - SECTION("issue #3204 - ambiguous regression") - { - const for_3204_bar bar_from_foo([](for_3204_foo) noexcept {}); // NOLINT(performance-unnecessary-value-param) - const for_3204_bar bar_from_json([](json) noexcept {}); // NOLINT(performance-unnecessary-value-param) - - CHECK(bar_from_foo.constructed_from == for_3204_bar::constructed_from_foo); - CHECK(bar_from_json.constructed_from == for_3204_bar::constructed_from_json); - } - - SECTION("issue #3333 - Ambiguous conversion from nlohmann::basic_json<> to custom class") - { - const json j - { - {"x", 1}, - {"y", 2} - }; - const for_3333 p = j; - - CHECK(p.x == 1); - CHECK(p.y == 2); - } - - SECTION("issue #3810 - ordered_json doesn't support construction from C array of custom type") - { - Example_3810 states[45]; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) - - // fix "not used" warning - states[0].bla = 1; - - const auto* const expected = R"([{"bla":1},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0}])"; - - // This works: - nlohmann::json j; - j["test"] = states; - CHECK(j["test"].dump() == expected); - - // This doesn't compile: - nlohmann::ordered_json oj; - oj["test"] = states; - CHECK(oj["test"].dump() == expected); - } - -#ifdef JSON_HAS_CPP_17 - SECTION("issue #4740 - build issue with std::optional") - { - const auto t1 = Example_4740(); - const auto j1 = nlohmann::json(t1); - CHECK(j1.dump() == "{\"host\":null,\"port\":null}"); - const auto t2 = j1.get(); - CHECK(!t2.host.has_value()); - CHECK(!t2.port.has_value()); - - // improve coverage - auto t3 = Example_4740(); - t3.port = 80; - t3.host = "example.com"; - const auto j2 = nlohmann::json(t3); - CHECK(j2.dump() == "{\"host\":\"example.com\",\"port\":80}"); - const auto t4 = j2.get(); - CHECK(t4.host.has_value()); - CHECK(t4.port.has_value()); - } -#endif - -#if !defined(_MSVC_LANG) - // MSVC returns garbage on invalid enum values, so this test is excluded - // there. - SECTION("issue #4762 - json exception 302 with unhelpful explanation : type must be number, but is number") - { - // In #4762, the main issue was that a json object with an invalid type - // returned "number" as type_name(), because this was the default case. - // This test makes sure we now return "invalid" instead. - json j; - j.m_data.m_type = static_cast(100); // NOLINT(clang-analyzer-optin.core.EnumCastOutOfRange) - CHECK(j.type_name() == "invalid"); - } -#endif - -#ifdef JSON_HAS_CPP_17 - SECTION("issue #4804: from_cbor incompatible with std::vector as binary_t") - { - const std::vector data = {0x80}; - const auto decoded = json_4804::from_cbor(data); - CHECK((decoded == json_4804::array())); - } - - SECTION("discussion #4209 - custom BinaryType direct assignment and round-tripping") - { - // Test that assigning a custom BinaryType directly creates a binary value, not an array - const std::vector original{std::byte{1}, std::byte{2}, std::byte{3}}; - const json_4804 j = original; - CHECK(j.is_binary()); - CHECK(!j.is_array()); - - // Test round-tripping: extracting the binary value back as the custom container type - const auto extracted = j.get>(); - CHECK(extracted == original); - - // Test that the default json alias behavior is unchanged: std::vector -> array - const json default_json = std::vector {1, 2, 3}; - CHECK(default_json.is_array()); - CHECK(!default_json.is_binary()); - } - - SECTION("discussion #4209 - custom BinaryType extraction from parsed array") - { - // Test that extracting a custom BinaryType from a parsed JSON array still works - // (not just from a binary-typed node) - const auto j = json_4804::parse("[1,2,3]"); - CHECK(j.is_array()); - CHECK(!j.is_binary()); - - // Extracting as custom BinaryType should work from arrays - const auto extracted = j.get>(); - CHECK(extracted.size() == 3); - CHECK(extracted[0] == std::byte{1}); - CHECK(extracted[1] == std::byte{2}); - CHECK(extracted[2] == std::byte{3}); - } - - SECTION("issue #5046 - implicit conversion of return json to std::optional no longer implicit") - { - const json jval{}; - auto GetValue = [](const json & valRoot) -> std::optional - { - if (valRoot.contains("default")) - { - return valRoot.at("default"); - } - return std::nullopt; - }; - auto result = GetValue(jval); - CHECK(!result.has_value()); - } -#endif - -#if JSON_HAS_RANGES == 1 - SECTION("issue #4440 - assert when using std::views::filter and GCC 10") - { - auto noOpFilter = std::views::filter([](auto&&) noexcept - { - return true; - }); - json j = {1, 2, 3}; - auto filtered = j | noOpFilter; - CHECK(*filtered.begin() == 1); - } -#endif - -#if JSON_HAS_RANGES && !defined(__MINGW32__) - SECTION("issue #4916 - constructing array from C++20 ranges view does not work") - { - std::vector nums{1, 2, 37, 42, 21}; - auto filteredNums = nums | std::views::filter([](int i) - { - return i > 10; - }); - json const j(filteredNums); - CHECK(j.type() == json::value_t::array); - CHECK(j == json({37, 42, 21})); - } -#endif - - // owning_view is not available in libstdc++ < 12 -#if JSON_HAS_RANGES && !defined(__MINGW32__) && !(defined(__GLIBCXX__) && _GLIBCXX_RELEASE < 12) - SECTION("issue #4916 - constructing array from prvalue C++20 ranges view (owning_view)") - { - json const j(std::vector {1, 2, 37, 42, 21} | std::views::filter([](int i) - { - return i > 10; - })); - CHECK(j.type() == json::value_t::array); - CHECK(j == json({37, 42, 21})); - } -#endif - -#if JSON_HAS_RANGES && !defined(__MINGW32__) - SECTION("issue #4916 - constructing array from C++20 transform view (prvalue elements)") - { - std::vector nums{1, 2, 3}; - auto t = nums | std::views::transform([](int i) noexcept - { - return i * 2; - }); - json const j(t); - CHECK(j.type() == json::value_t::array); - CHECK(j == json({2, 4, 6})); - } -#endif -} - -TEST_CASE_TEMPLATE("issue #4798 - nlohmann::json::to_msgpack() encode float NaN as double", T, double, float) // NOLINT(readability-math-missing-parentheses, bugprone-throwing-static-initialization) -{ - // With issue #4798, we encode NaN, infinity, and -infinity as float instead - // of double to allow for smaller encodings. - const json jx = std::numeric_limits::quiet_NaN(); - const json jy = std::numeric_limits::infinity(); - const json jz = -std::numeric_limits::infinity(); - - ///////////////////////////////////////////////////////////////////////// - // MessagePack - ///////////////////////////////////////////////////////////////////////// - - // expected MessagePack values - const std::vector msgpack_x = {{0xCA, 0x7F, 0xC0, 0x00, 0x00}}; - const std::vector msgpack_y = {{0xCA, 0x7F, 0x80, 0x00, 0x00}}; - const std::vector msgpack_z = {{0xCA, 0xFF, 0x80, 0x00, 0x00}}; - - CHECK(json::to_msgpack(jx) == msgpack_x); - CHECK(json::to_msgpack(jy) == msgpack_y); - CHECK(json::to_msgpack(jz) == msgpack_z); - - CHECK(std::isnan(json::from_msgpack(msgpack_x).get())); - CHECK(json::from_msgpack(msgpack_y).get() == std::numeric_limits::infinity()); - CHECK(json::from_msgpack(msgpack_z).get() == -std::numeric_limits::infinity()); - - // Make sure the other MessagePakc encodings for NaN, infinity, and - // -infinity are still supported. - const std::vector msgpack_x_2 = {{0xCB, 0x7F, 0xF8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}}; - const std::vector msgpack_y_2 = {{0xCB, 0x7F, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}}; - const std::vector msgpack_z_2 = {{0xCB, 0xFF, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}}; - CHECK(std::isnan(json::from_msgpack(msgpack_x_2).get())); - CHECK(json::from_msgpack(msgpack_y_2).get() == std::numeric_limits::infinity()); - CHECK(json::from_msgpack(msgpack_z_2).get() == -std::numeric_limits::infinity()); - - ///////////////////////////////////////////////////////////////////////// - // CBOR - ///////////////////////////////////////////////////////////////////////// - - // expected CBOR values - const std::vector cbor_x = {{0xF9, 0x7E, 0x00}}; - const std::vector cbor_y = {{0xF9, 0x7C, 0x00}}; - const std::vector cbor_z = {{0xF9, 0xfC, 0x00}}; - - CHECK(json::to_cbor(jx) == cbor_x); - CHECK(json::to_cbor(jy) == cbor_y); - CHECK(json::to_cbor(jz) == cbor_z); - - CHECK(std::isnan(json::from_cbor(cbor_x).get())); - CHECK(json::from_cbor(cbor_y).get() == std::numeric_limits::infinity()); - CHECK(json::from_cbor(cbor_z).get() == -std::numeric_limits::infinity()); - - // Make sure the other CBOR encodings for NaN, infinity, and -infinity are - // still supported. - const std::vector cbor_x_2 = {{0xFA, 0x7F, 0xC0, 0x00, 0x00}}; - const std::vector cbor_y_2 = {{0xFA, 0x7F, 0x80, 0x00, 0x00}}; - const std::vector cbor_z_2 = {{0xFA, 0xFF, 0x80, 0x00, 0x00}}; - const std::vector cbor_x_3 = {{0xFB, 0x7F, 0xF8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}}; - const std::vector cbor_y_3 = {{0xFB, 0x7F, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}}; - const std::vector cbor_z_3 = {{0xFB, 0xFF, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}}; - CHECK(std::isnan(json::from_cbor(cbor_x_2).get())); - CHECK(json::from_cbor(cbor_y_2).get() == std::numeric_limits::infinity()); - CHECK(json::from_cbor(cbor_z_2).get() == -std::numeric_limits::infinity()); - CHECK(std::isnan(json::from_cbor(cbor_x_3).get())); - CHECK(json::from_cbor(cbor_y_3).get() == std::numeric_limits::infinity()); - CHECK(json::from_cbor(cbor_z_3).get() == -std::numeric_limits::infinity()); -} - -TEST_CASE("regression test #5074 - portable workaround for single-element brace init") -{ - json const j_obj = {{"key", "value"}}; - - json const j = json::array({j_obj}); - CHECK(j.is_array()); - CHECK(j.size() == 1); - CHECK(j[0] == j_obj); -} - -#if defined(JSON_BRACE_INIT_COPY_SEMANTICS) && (JSON_BRACE_INIT_COPY_SEMANTICS == 1) -TEST_CASE("regression test #5074 - single-element brace init with JSON_BRACE_INIT_COPY_SEMANTICS") -{ - // with JSON_BRACE_INIT_COPY_SEMANTICS: single-element brace init copies/moves - json const j_obj = {{"key", "value"}, {"num", 42}}; - json const j_arr = {1, 2, 3}; - - // object: brace init copies instead of wrapping - json const j1{j_obj}; - CHECK(j1.is_object()); - CHECK(j1 == j_obj); - - // array: brace init copies instead of wrapping - json const j2{j_arr}; - CHECK(j2.is_array()); - CHECK(j2.size() == 3); - CHECK(j2 == j_arr); - - // primitives still work as initializer lists - json const j3{true}; - CHECK(j3.is_boolean()); - - json const j4{42}; - CHECK(j4.is_number_integer()); -} -#endif - -struct Example_5122 -{ - float b = 2; - nlohmann::ordered_map c{}; // NOLINT(readability-redundant-member-init): needed for GCC -Weffc++ - int a = 1; - NLOHMANN_DEFINE_TYPE_INTRUSIVE_WITH_DEFAULT(Example_5122, b, c, a) -}; - -TEST_CASE("regression test #5122 - from_json into types holding nlohmann::ordered_map") -{ - Example_5122 src; - src.c.emplace("first", "1"); - src.c.emplace("second", "2"); - - ordered_json const j = src; - Example_5122 const dst = j.get(); - - CHECK(dst.b == src.b); - CHECK(dst.a == src.a); - REQUIRE(dst.c.size() == src.c.size()); - auto src_it = src.c.begin(); - auto dst_it = dst.c.begin(); - for (; src_it != src.c.end(); ++src_it, ++dst_it) - { - CHECK(dst_it->first == src_it->first); - CHECK(dst_it->second == src_it->second); - } -} - -// -Wself-assign-overloaded was introduced in Clang 7. Gate the pragma on -// __has_warning so older Clang versions do not error with "unknown warning -// group". The __has_warning check has to stay inside the __clang__ branch -// because GCC does not provide it and would tokenize-error on the argument. -#if defined(__clang__) && defined(__has_warning) - #if __has_warning("-Wself-assign-overloaded") - DOCTEST_CLANG_SUPPRESS_WARNING_PUSH - DOCTEST_CLANG_SUPPRESS_WARNING("-Wself-assign-overloaded") - #endif -#endif - -TEST_CASE("regression test #5122 - nlohmann::ordered_map copy-assignment is self-assignment safe") -{ - nlohmann::ordered_map m; - m.emplace("first", "1"); - m.emplace("second", "2"); - - // Insertion order is preserved by ordered_map, so we can check it directly. - m = m; - - REQUIRE(m.size() == 2); - auto it = m.begin(); - CHECK(it->first == "first"); - CHECK(it->second == "1"); - ++it; - CHECK(it->first == "second"); - CHECK(it->second == "2"); -} - -#if defined(__clang__) && defined(__has_warning) - #if __has_warning("-Wself-assign-overloaded") - DOCTEST_CLANG_SUPPRESS_WARNING_POP - #endif -#endif - -TEST_CASE("regression test #5122 - nlohmann::ordered_map move-assignment transfers contents") -{ - nlohmann::ordered_map src; - src.emplace("first", "1"); - src.emplace("second", "2"); - - nlohmann::ordered_map dst; - dst.emplace("stale", "x"); - dst = std::move(src); - - REQUIRE(dst.size() == 2); - auto it = dst.begin(); - CHECK(it->first == "first"); - CHECK(it->second == "1"); - ++it; - CHECK(it->first == "second"); - CHECK(it->second == "2"); - - // Re-assigning into the moved-from object must leave it in a usable state. - src = nlohmann::ordered_map {}; - src.emplace("after-move", "3"); - REQUIRE(src.size() == 1); - CHECK(src.begin()->first == "after-move"); -} - -// Stand-in for a third-party library (e.g., Eigen as of 3.4, which added -// STL-compatible begin()/end() to its vector types), living in its own -// namespace with its own to_json overload for its vector type. -namespace issue_4320_eigen -{ -// "array-compatible" from the library's point of view (it has begin()/end()), -// but for which this (fake) third-party namespace provides its own to_json. -struct vector3 -{ - double v[3]; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays,cppcoreguidelines-use-default-member-init,modernize-use-default-member-init) - vector3(double x, double y, double z) : v{x, y, z} {} // NOLINT(hicpp-member-init,cppcoreguidelines-pro-type-member-init) - double x() const - { - return v[0]; - } - double y() const - { - return v[1]; - } - double z() const - { - return v[2]; - } - double* begin() - { - return v; - } - double* end() - { - return v + 3; - } - const double* begin() const - { - return v; - } - const double* end() const - { - return v + 3; - } -}; - -inline void to_json(json& j, const vector3& v) // NOLINT(misc-use-internal-linkage) -{ - j = {{"x", v.x()}, {"y", v.y()}, {"z", v.z()}}; -} -} // namespace issue_4320_eigen - -// The user's own namespace, using the (fake) Eigen type as an implementation -// detail behind a payload type that has nothing to do with vectors/arrays. -namespace issue_4320 -{ -// Publicly derives from issue_4320_eigen::vector3 but does *not* define its -// own to_json - it is only ever used as a temporary to reach the base -// class's to_json via ADL. -struct vector3_wrapper : issue_4320_eigen::vector3 -{ - using issue_4320_eigen::vector3::vector3; -}; - -struct payload -{ - double x, y, z; -}; - -inline vector3_wrapper to_eigen(const payload& p) // NOLINT(misc-use-internal-linkage) -{ - return {p.x, p.y, p.z}; -} - -inline void to_json(json& j, const payload& p) // NOLINT(misc-use-internal-linkage) -{ - // Unqualified call, passing a *derived* vector3_wrapper: relies on ADL - // finding issue_4320_eigen::to_json(json&, const vector3&) through the - // vector3 base class, via a derived-to-base conversion. Must NOT resolve - // to the library's own generic array-compatible to_json (an exact-match - // template for vector3_wrapper, since it also has begin()/end()), which - // would serialize this as [x, y, z] instead of {"x":x, "y":y, "z":z}. - to_json(j, to_eigen(p)); -} -} // namespace issue_4320 - -TEST_CASE("issue #4320 - custom base class must not leak nlohmann::detail into ADL") -{ - // Before the fix, basic_json unconditionally derived from a type living in - // nlohmann::detail (json_default_base), which made nlohmann::detail an - // associated namespace of every basic_json for ADL purposes. That leaked - // the library's internal generic-array to_json overload into unqualified - // to_json() calls made from user code, silently bypassing user-defined - // to_json overloads reached via a derived-to-base conversion. - const issue_4320::payload p{1.0, 2.0, 3.0}; - - json j; - to_json(j, p); - CHECK(j == json({{"x", 1.0}, {"y", 2.0}, {"z", 3.0}})); -} - -TEST_CASE("issue #5338 - truncated CBOR tagged binary subtype is rejected") -{ - const std::vector> truncated_tags = - { - {0xD8}, - {0xD9, 0x00}, - {0xDA, 0x00, 0x00, 0x00}, - {0xDB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00} - }; - - for (const auto& data : truncated_tags) - { - CAPTURE(data); - for (const auto tag_handler : - { - json::cbor_tag_handler_t::ignore, json::cbor_tag_handler_t::store - }) - { - CAPTURE(tag_handler); - const auto result = json::from_cbor(data, true, false, tag_handler); - CHECK(result.is_discarded()); - } - } -} - -TEST_CASE("issue #5402 - update(merge_objects=true) overwrites a primitive with an object") -{ - json t = {{"k", 1}}; - t.update(json{{"k", {{"x", 2}}}}, true); - CHECK(t == json({{"k", {{"x", 2}}}})); - - json mixed = {{"keep", {{"a", 1}}}, {"replace", 1}}; - mixed.update(json{{"keep", {{"b", 2}}}, {"replace", {{"x", 2}}}}, true); - CHECK(mixed == json({{"keep", {{"a", 1}, {"b", 2}}}, {"replace", {{"x", 2}}}})); } DOCTEST_CLANG_SUPPRESS_WARNING_POP diff --git a/tests/src/unit-regression3.cpp b/tests/src/unit-regression3.cpp new file mode 100644 index 000000000..cb2ed59a6 --- /dev/null +++ b/tests/src/unit-regression3.cpp @@ -0,0 +1,899 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +// cmake/test.cmake selects the C++ standard versions with which to build a +// unit test based on the presence of JSON_HAS_CPP_ macros. +// When using macros that are only defined for particular versions of the standard +// (e.g., JSON_HAS_FILESYSTEM for C++17 and up), please mention the corresponding +// version macro in a comment close by, like this: +// JSON_HAS_CPP_ (do not remove; see note at top of file) + +#include "doctest_compatibility.h" + +// for some reason including this after the json header leads to linker errors with VS 2017... +#include + +#define JSON_TESTS_PRIVATE +#include +using json = nlohmann::json; +using ordered_json = nlohmann::ordered_json; +#ifdef JSON_TEST_NO_GLOBAL_UDLS + using namespace nlohmann::literals; // NOLINT(google-build-using-namespace) +#endif + +#include +#include +#include +#include + +#ifdef JSON_HAS_CPP_17 + #include + #include +#endif + +#ifdef JSON_HAS_CPP_17 + #if __has_include() + #include + #elif __has_include() + #endif + + ///////////////////////////////////////////////////////////////////// + // for #4804 + ///////////////////////////////////////////////////////////////////// + using json_4804 = nlohmann::basic_json, // BinaryType + void // CustomBaseClass + >; +#endif + +#ifdef JSON_HAS_CPP_20 + #if __has_include() + #include + #endif +#endif + +///////////////////////////////////////////////////////////////////// +// for #4825 - explicitly instantiating basic_json must compile; this +// forces instantiation of binary_writer::write_bjdata_ndarray, whose +// static_cast was ambiguous under explicit instantiation on +// C++17. Merely compiling this translation unit is the regression test. +///////////////////////////////////////////////////////////////////// +template class nlohmann::basic_json<>; + +///////////////////////////////////////////////////////////////////// +// for #4440 +///////////////////////////////////////////////////////////////////// +#if JSON_HAS_RANGES == 1 + #include +#endif + +// NLOHMANN_JSON_SERIALIZE_ENUM uses a static std::pair +DOCTEST_CLANG_SUPPRESS_WARNING_PUSH +DOCTEST_CLANG_SUPPRESS_WARNING("-Wexit-time-destructors") +///////////////////////////////////////////////////////////////////// +// for #3077 +///////////////////////////////////////////////////////////////////// + +class FooAlloc +{}; + +class Foo +{ + public: + explicit Foo(const FooAlloc& /* unused */ = FooAlloc()) {} + + bool value = false; +}; + +class FooBar +{ + public: + Foo foo{}; // NOLINT(readability-redundant-member-init) +}; + +inline void from_json(const nlohmann::json& j, FooBar& fb) // NOLINT(misc-use-internal-linkage) +{ + j.at("value").get_to(fb.foo.value); +} + +///////////////////////////////////////////////////////////////////// +// for #3171 +///////////////////////////////////////////////////////////////////// + +struct for_3171_base // NOLINT(cppcoreguidelines-special-member-functions) +{ + for_3171_base(const std::string& /*unused*/ = {}) {} + virtual ~for_3171_base(); + + for_3171_base(const for_3171_base& other) // NOLINT(hicpp-use-equals-default,modernize-use-equals-default) + : str(other.str) + {} + + for_3171_base& operator=(const for_3171_base& other) + { + if (this != &other) + { + str = other.str; + } + return *this; + } + + for_3171_base(for_3171_base&& other) noexcept + : str(std::move(other.str)) + {} + + for_3171_base& operator=(for_3171_base&& other) noexcept + { + if (this != &other) + { + str = std::move(other.str); + } + return *this; + } + + virtual void _from_json(const json& j) + { + j.at("str").get_to(str); + } + + std::string str{}; // NOLINT(readability-redundant-member-init) +}; + +for_3171_base::~for_3171_base() = default; + +struct for_3171_derived : public for_3171_base +{ + for_3171_derived() = default; + ~for_3171_derived() override; + explicit for_3171_derived(const std::string& /*unused*/) { } + + for_3171_derived(const for_3171_derived& other) // NOLINT(hicpp-use-equals-default,modernize-use-equals-default) + : for_3171_base(other) + {} + + for_3171_derived& operator=(const for_3171_derived& other) + { + if (this != &other) + { + for_3171_base::operator=(other); // Call base class assignment operator + } + return *this; + } + + for_3171_derived(for_3171_derived&& other) noexcept + : for_3171_base(std::move(other)) + {} + + for_3171_derived& operator=(for_3171_derived&& other) noexcept + { + if (this != &other) + { + for_3171_base::operator=(std::move(other)); // Call base class move assignment operator + } + return *this; + } +}; + +for_3171_derived::~for_3171_derived() = default; + +inline void from_json(const json& j, for_3171_base& tb) // NOLINT(misc-use-internal-linkage) +{ + tb._from_json(j); +} + +///////////////////////////////////////////////////////////////////// +// for #3312 +///////////////////////////////////////////////////////////////////// + +#ifdef JSON_HAS_CPP_20 +struct for_3312 +{ + std::string name; +}; + +inline void from_json(const json& j, for_3312& obj) // NOLINT(misc-use-internal-linkage) +{ + j.at("name").get_to(obj.name); +} +#endif + +///////////////////////////////////////////////////////////////////// +// for #3204 +///////////////////////////////////////////////////////////////////// + +struct for_3204_foo +{ + for_3204_foo() = default; + explicit for_3204_foo(std::string /*unused*/) {} // NOLINT(performance-unnecessary-value-param) +}; + +struct for_3204_bar +{ + enum constructed_from_t // NOLINT(cppcoreguidelines-use-enum-class) + { + constructed_from_none = 0, + constructed_from_foo = 1, + constructed_from_json = 2 + }; + + explicit for_3204_bar(std::function /*unused*/) noexcept // NOLINT(performance-unnecessary-value-param) + : constructed_from(constructed_from_foo) {} + explicit for_3204_bar(std::function /*unused*/) noexcept // NOLINT(performance-unnecessary-value-param) + : constructed_from(constructed_from_json) {} + + constructed_from_t constructed_from = constructed_from_none; +}; + +///////////////////////////////////////////////////////////////////// +// for #3333 +///////////////////////////////////////////////////////////////////// + +struct for_3333 final +{ + for_3333(int x_ = 0, int y_ = 0) : x(x_), y(y_) {} + + template + for_3333(const T& /*unused*/) + { + CHECK(false); + } + + int x = 0; + int y = 0; +}; + +template <> +inline for_3333::for_3333(const json& j) + : for_3333(j.value("x", 0), j.value("y", 0)) +{} + +///////////////////////////////////////////////////////////////////// +// for #3810 +///////////////////////////////////////////////////////////////////// + +struct Example_3810 +{ + int bla{}; + + Example_3810() = default; +}; + +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Example_3810, bla) // NOLINT(misc-use-internal-linkage) + +///////////////////////////////////////////////////////////////////// +// for #4740 +///////////////////////////////////////////////////////////////////// + +#ifdef JSON_HAS_CPP_17 +struct Example_4740 +{ + std::optional host = std::nullopt; + std::optional port = std::nullopt; + NLOHMANN_DEFINE_TYPE_INTRUSIVE_WITH_DEFAULT(Example_4740, host, port) +}; +#endif + +TEST_CASE("regression tests 3") +{ +#if JSON_HAS_FILESYSTEM || JSON_HAS_EXPERIMENTAL_FILESYSTEM + // JSON_HAS_CPP_17 (do not remove; see note at top of file) + SECTION("issue #3070 - Version 3.10.3 breaks backward-compatibility with 3.10.2 ") + { + nlohmann::detail::std_fs::path text_path("/tmp/text.txt"); + const json j(text_path); + + const auto j_path = j.get(); + CHECK(j_path == text_path); + +#if DOCTEST_CLANG || DOCTEST_GCC >= DOCTEST_COMPILER(8, 4, 0) + // only known to work on Clang and GCC >=8.4 + CHECK_THROWS_WITH_AS(nlohmann::detail::std_fs::path(json(1)), "[json.exception.type_error.302] type must be string, but is number", json::type_error); +#endif + } +#endif + + SECTION("issue #3077 - explicit constructor with default does not compile") + { + json j; + j[0]["value"] = true; + std::vector foo; + j.get_to(foo); + } + + SECTION("issue #3108 - ordered_json doesn't support range based erase") + { + ordered_json j = {1, 2, 2, 4}; + + auto last = std::unique(j.begin(), j.end()); + j.erase(last, j.end()); + + CHECK(j.dump() == "[1,2,4]"); + + j.erase(std::remove_if(j.begin(), j.end(), [](const ordered_json & val) + { + return val == 2; + }), j.end()); + + CHECK(j.dump() == "[1,4]"); + } + + SECTION("issue #3343 - json and ordered_json are not interchangeable") + { + json::object_t jobj({ { "product", "one" } }); + ordered_json::object_t ojobj({{"product", "one"}}); + + auto jit = jobj.begin(); + auto ojit = ojobj.begin(); + + CHECK(jit->first == ojit->first); + CHECK(jit->second.get() == ojit->second.get()); + } + + SECTION("issue #3171 - if class is_constructible from std::string wrong from_json overload is being selected, compilation failed") + { + const json j{{ "str", "value"}}; + + // failed with: error: no match for ‘operator=’ (operand types are ‘for_3171_derived’ and ‘const nlohmann::basic_json<>::string_t’ + // {aka ‘const std::__cxx11::basic_string’}) + // s = *j.template get_ptr(); + auto td = j.get(); + + CHECK(td.str == "value"); + } + +#ifdef JSON_HAS_CPP_20 + SECTION("issue #3312 - Parse to custom class from unordered_json breaks on G++11.2.0 with C++20") + { + // see test for #3171 + const ordered_json j = {{"name", "class"}}; + for_3312 obj{}; + + j.get_to(obj); + + CHECK(obj.name == "class"); + } +#endif + +#if defined(JSON_HAS_CPP_17) && JSON_USE_IMPLICIT_CONVERSIONS + SECTION("issue #3428 - Error occurred when converting nlohmann::json to std::any") + { + const json j; + const std::any a1 = j; + std::any&& a2 = j; + + CHECK(a1.type() == typeid(j)); + CHECK(a2.type() == typeid(j)); + } +#endif + + SECTION("issue #3204 - ambiguous regression") + { + const for_3204_bar bar_from_foo([](for_3204_foo) noexcept {}); // NOLINT(performance-unnecessary-value-param) + const for_3204_bar bar_from_json([](json) noexcept {}); // NOLINT(performance-unnecessary-value-param) + + CHECK(bar_from_foo.constructed_from == for_3204_bar::constructed_from_foo); + CHECK(bar_from_json.constructed_from == for_3204_bar::constructed_from_json); + } + + SECTION("issue #3333 - Ambiguous conversion from nlohmann::basic_json<> to custom class") + { + const json j + { + {"x", 1}, + {"y", 2} + }; + const for_3333 p = j; + + CHECK(p.x == 1); + CHECK(p.y == 2); + } + + SECTION("issue #3810 - ordered_json doesn't support construction from C array of custom type") + { + Example_3810 states[45]; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + + // fix "not used" warning + states[0].bla = 1; + + const auto* const expected = R"([{"bla":1},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0},{"bla":0}])"; + + // This works: + nlohmann::json j; + j["test"] = states; + CHECK(j["test"].dump() == expected); + + // This doesn't compile: + nlohmann::ordered_json oj; + oj["test"] = states; + CHECK(oj["test"].dump() == expected); + } + +#ifdef JSON_HAS_CPP_17 + SECTION("issue #4740 - build issue with std::optional") + { + const auto t1 = Example_4740(); + const auto j1 = nlohmann::json(t1); + CHECK(j1.dump() == "{\"host\":null,\"port\":null}"); + const auto t2 = j1.get(); + CHECK(!t2.host.has_value()); + CHECK(!t2.port.has_value()); + + // improve coverage + auto t3 = Example_4740(); + t3.port = 80; + t3.host = "example.com"; + const auto j2 = nlohmann::json(t3); + CHECK(j2.dump() == "{\"host\":\"example.com\",\"port\":80}"); + const auto t4 = j2.get(); + CHECK(t4.host.has_value()); + CHECK(t4.port.has_value()); + } +#endif + +#if !defined(_MSVC_LANG) + // MSVC returns garbage on invalid enum values, so this test is excluded + // there. + SECTION("issue #4762 - json exception 302 with unhelpful explanation : type must be number, but is number") + { + // In #4762, the main issue was that a json object with an invalid type + // returned "number" as type_name(), because this was the default case. + // This test makes sure we now return "invalid" instead. + json j; + j.m_data.m_type = static_cast(100); // NOLINT(clang-analyzer-optin.core.EnumCastOutOfRange) + CHECK(j.type_name() == "invalid"); + } +#endif + +#ifdef JSON_HAS_CPP_17 + SECTION("issue #4804: from_cbor incompatible with std::vector as binary_t") + { + const std::vector data = {0x80}; + const auto decoded = json_4804::from_cbor(data); + CHECK((decoded == json_4804::array())); + } + + SECTION("discussion #4209 - custom BinaryType direct assignment and round-tripping") + { + // Test that assigning a custom BinaryType directly creates a binary value, not an array + const std::vector original{std::byte{1}, std::byte{2}, std::byte{3}}; + const json_4804 j = original; + CHECK(j.is_binary()); + CHECK(!j.is_array()); + + // Test round-tripping: extracting the binary value back as the custom container type + const auto extracted = j.get>(); + CHECK(extracted == original); + + // Test that the default json alias behavior is unchanged: std::vector -> array + const json default_json = std::vector {1, 2, 3}; + CHECK(default_json.is_array()); + CHECK(!default_json.is_binary()); + } + + SECTION("discussion #4209 - custom BinaryType extraction from parsed array") + { + // Test that extracting a custom BinaryType from a parsed JSON array still works + // (not just from a binary-typed node) + const auto j = json_4804::parse("[1,2,3]"); + CHECK(j.is_array()); + CHECK(!j.is_binary()); + + // Extracting as custom BinaryType should work from arrays + const auto extracted = j.get>(); + CHECK(extracted.size() == 3); + CHECK(extracted[0] == std::byte{1}); + CHECK(extracted[1] == std::byte{2}); + CHECK(extracted[2] == std::byte{3}); + } + + SECTION("issue #5046 - implicit conversion of return json to std::optional no longer implicit") + { + const json jval{}; + auto GetValue = [](const json & valRoot) -> std::optional + { + if (valRoot.contains("default")) + { + return valRoot.at("default"); + } + return std::nullopt; + }; + auto result = GetValue(jval); + CHECK(!result.has_value()); + } +#endif + +#if JSON_HAS_RANGES == 1 + SECTION("issue #4440 - assert when using std::views::filter and GCC 10") + { + auto noOpFilter = std::views::filter([](auto&&) noexcept + { + return true; + }); + json j = {1, 2, 3}; + auto filtered = j | noOpFilter; + CHECK(*filtered.begin() == 1); + } +#endif + +#if JSON_HAS_RANGES && !defined(__MINGW32__) + SECTION("issue #4916 - constructing array from C++20 ranges view does not work") + { + std::vector nums{1, 2, 37, 42, 21}; + auto filteredNums = nums | std::views::filter([](int i) + { + return i > 10; + }); + json const j(filteredNums); + CHECK(j.type() == json::value_t::array); + CHECK(j == json({37, 42, 21})); + } +#endif + + // owning_view is not available in libstdc++ < 12 +#if JSON_HAS_RANGES && !defined(__MINGW32__) && !(defined(__GLIBCXX__) && _GLIBCXX_RELEASE < 12) + SECTION("issue #4916 - constructing array from prvalue C++20 ranges view (owning_view)") + { + json const j(std::vector {1, 2, 37, 42, 21} | std::views::filter([](int i) + { + return i > 10; + })); + CHECK(j.type() == json::value_t::array); + CHECK(j == json({37, 42, 21})); + } +#endif + +#if JSON_HAS_RANGES && !defined(__MINGW32__) + SECTION("issue #4916 - constructing array from C++20 transform view (prvalue elements)") + { + std::vector nums{1, 2, 3}; + auto t = nums | std::views::transform([](int i) noexcept + { + return i * 2; + }); + json const j(t); + CHECK(j.type() == json::value_t::array); + CHECK(j == json({2, 4, 6})); + } +#endif +} + +TEST_CASE_TEMPLATE("issue #4798 - nlohmann::json::to_msgpack() encode float NaN as double", T, double, float) // NOLINT(readability-math-missing-parentheses, bugprone-throwing-static-initialization) +{ + // With issue #4798, we encode NaN, infinity, and -infinity as float instead + // of double to allow for smaller encodings. + const json jx = std::numeric_limits::quiet_NaN(); + const json jy = std::numeric_limits::infinity(); + const json jz = -std::numeric_limits::infinity(); + + ///////////////////////////////////////////////////////////////////////// + // MessagePack + ///////////////////////////////////////////////////////////////////////// + + // expected MessagePack values + const std::vector msgpack_x = {{0xCA, 0x7F, 0xC0, 0x00, 0x00}}; + const std::vector msgpack_y = {{0xCA, 0x7F, 0x80, 0x00, 0x00}}; + const std::vector msgpack_z = {{0xCA, 0xFF, 0x80, 0x00, 0x00}}; + + CHECK(json::to_msgpack(jx) == msgpack_x); + CHECK(json::to_msgpack(jy) == msgpack_y); + CHECK(json::to_msgpack(jz) == msgpack_z); + + CHECK(std::isnan(json::from_msgpack(msgpack_x).get())); + CHECK(json::from_msgpack(msgpack_y).get() == std::numeric_limits::infinity()); + CHECK(json::from_msgpack(msgpack_z).get() == -std::numeric_limits::infinity()); + + // Make sure the other MessagePakc encodings for NaN, infinity, and + // -infinity are still supported. + const std::vector msgpack_x_2 = {{0xCB, 0x7F, 0xF8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}}; + const std::vector msgpack_y_2 = {{0xCB, 0x7F, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}}; + const std::vector msgpack_z_2 = {{0xCB, 0xFF, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}}; + CHECK(std::isnan(json::from_msgpack(msgpack_x_2).get())); + CHECK(json::from_msgpack(msgpack_y_2).get() == std::numeric_limits::infinity()); + CHECK(json::from_msgpack(msgpack_z_2).get() == -std::numeric_limits::infinity()); + + ///////////////////////////////////////////////////////////////////////// + // CBOR + ///////////////////////////////////////////////////////////////////////// + + // expected CBOR values + const std::vector cbor_x = {{0xF9, 0x7E, 0x00}}; + const std::vector cbor_y = {{0xF9, 0x7C, 0x00}}; + const std::vector cbor_z = {{0xF9, 0xfC, 0x00}}; + + CHECK(json::to_cbor(jx) == cbor_x); + CHECK(json::to_cbor(jy) == cbor_y); + CHECK(json::to_cbor(jz) == cbor_z); + + CHECK(std::isnan(json::from_cbor(cbor_x).get())); + CHECK(json::from_cbor(cbor_y).get() == std::numeric_limits::infinity()); + CHECK(json::from_cbor(cbor_z).get() == -std::numeric_limits::infinity()); + + // Make sure the other CBOR encodings for NaN, infinity, and -infinity are + // still supported. + const std::vector cbor_x_2 = {{0xFA, 0x7F, 0xC0, 0x00, 0x00}}; + const std::vector cbor_y_2 = {{0xFA, 0x7F, 0x80, 0x00, 0x00}}; + const std::vector cbor_z_2 = {{0xFA, 0xFF, 0x80, 0x00, 0x00}}; + const std::vector cbor_x_3 = {{0xFB, 0x7F, 0xF8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}}; + const std::vector cbor_y_3 = {{0xFB, 0x7F, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}}; + const std::vector cbor_z_3 = {{0xFB, 0xFF, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}}; + CHECK(std::isnan(json::from_cbor(cbor_x_2).get())); + CHECK(json::from_cbor(cbor_y_2).get() == std::numeric_limits::infinity()); + CHECK(json::from_cbor(cbor_z_2).get() == -std::numeric_limits::infinity()); + CHECK(std::isnan(json::from_cbor(cbor_x_3).get())); + CHECK(json::from_cbor(cbor_y_3).get() == std::numeric_limits::infinity()); + CHECK(json::from_cbor(cbor_z_3).get() == -std::numeric_limits::infinity()); +} + +TEST_CASE("regression test #5074 - portable workaround for single-element brace init") +{ + json const j_obj = {{"key", "value"}}; + + json const j = json::array({j_obj}); + CHECK(j.is_array()); + CHECK(j.size() == 1); + CHECK(j[0] == j_obj); +} + +#if defined(JSON_BRACE_INIT_COPY_SEMANTICS) && (JSON_BRACE_INIT_COPY_SEMANTICS == 1) +TEST_CASE("regression test #5074 - single-element brace init with JSON_BRACE_INIT_COPY_SEMANTICS") +{ + // with JSON_BRACE_INIT_COPY_SEMANTICS: single-element brace init copies/moves + json const j_obj = {{"key", "value"}, {"num", 42}}; + json const j_arr = {1, 2, 3}; + + // object: brace init copies instead of wrapping + json const j1{j_obj}; + CHECK(j1.is_object()); + CHECK(j1 == j_obj); + + // array: brace init copies instead of wrapping + json const j2{j_arr}; + CHECK(j2.is_array()); + CHECK(j2.size() == 3); + CHECK(j2 == j_arr); + + // primitives still work as initializer lists + json const j3{true}; + CHECK(j3.is_boolean()); + + json const j4{42}; + CHECK(j4.is_number_integer()); +} +#endif + +struct Example_5122 +{ + float b = 2; + nlohmann::ordered_map c{}; // NOLINT(readability-redundant-member-init): needed for GCC -Weffc++ + int a = 1; + NLOHMANN_DEFINE_TYPE_INTRUSIVE_WITH_DEFAULT(Example_5122, b, c, a) +}; + +TEST_CASE("regression test #5122 - from_json into types holding nlohmann::ordered_map") +{ + Example_5122 src; + src.c.emplace("first", "1"); + src.c.emplace("second", "2"); + + ordered_json const j = src; + Example_5122 const dst = j.get(); + + CHECK(dst.b == src.b); + CHECK(dst.a == src.a); + REQUIRE(dst.c.size() == src.c.size()); + auto src_it = src.c.begin(); + auto dst_it = dst.c.begin(); + for (; src_it != src.c.end(); ++src_it, ++dst_it) + { + CHECK(dst_it->first == src_it->first); + CHECK(dst_it->second == src_it->second); + } +} + +// -Wself-assign-overloaded was introduced in Clang 7. Gate the pragma on +// __has_warning so older Clang versions do not error with "unknown warning +// group". The __has_warning check has to stay inside the __clang__ branch +// because GCC does not provide it and would tokenize-error on the argument. +#if defined(__clang__) && defined(__has_warning) + #if __has_warning("-Wself-assign-overloaded") + DOCTEST_CLANG_SUPPRESS_WARNING_PUSH + DOCTEST_CLANG_SUPPRESS_WARNING("-Wself-assign-overloaded") + #endif +#endif + +TEST_CASE("regression test #5122 - nlohmann::ordered_map copy-assignment is self-assignment safe") +{ + nlohmann::ordered_map m; + m.emplace("first", "1"); + m.emplace("second", "2"); + + // Insertion order is preserved by ordered_map, so we can check it directly. + m = m; + + REQUIRE(m.size() == 2); + auto it = m.begin(); + CHECK(it->first == "first"); + CHECK(it->second == "1"); + ++it; + CHECK(it->first == "second"); + CHECK(it->second == "2"); +} + +#if defined(__clang__) && defined(__has_warning) + #if __has_warning("-Wself-assign-overloaded") + DOCTEST_CLANG_SUPPRESS_WARNING_POP + #endif +#endif + +TEST_CASE("regression test #5122 - nlohmann::ordered_map move-assignment transfers contents") +{ + nlohmann::ordered_map src; + src.emplace("first", "1"); + src.emplace("second", "2"); + + nlohmann::ordered_map dst; + dst.emplace("stale", "x"); + dst = std::move(src); + + REQUIRE(dst.size() == 2); + auto it = dst.begin(); + CHECK(it->first == "first"); + CHECK(it->second == "1"); + ++it; + CHECK(it->first == "second"); + CHECK(it->second == "2"); + + // Re-assigning into the moved-from object must leave it in a usable state. + src = nlohmann::ordered_map {}; + src.emplace("after-move", "3"); + REQUIRE(src.size() == 1); + CHECK(src.begin()->first == "after-move"); +} + +// Stand-in for a third-party library (e.g., Eigen as of 3.4, which added +// STL-compatible begin()/end() to its vector types), living in its own +// namespace with its own to_json overload for its vector type. +namespace issue_4320_eigen +{ +// "array-compatible" from the library's point of view (it has begin()/end()), +// but for which this (fake) third-party namespace provides its own to_json. +struct vector3 +{ + double v[3]; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays,cppcoreguidelines-use-default-member-init,modernize-use-default-member-init) + vector3(double x, double y, double z) : v{x, y, z} {} // NOLINT(hicpp-member-init,cppcoreguidelines-pro-type-member-init) + double x() const + { + return v[0]; + } + double y() const + { + return v[1]; + } + double z() const + { + return v[2]; + } + double* begin() + { + return v; + } + double* end() + { + return v + 3; + } + const double* begin() const + { + return v; + } + const double* end() const + { + return v + 3; + } +}; + +inline void to_json(json& j, const vector3& v) // NOLINT(misc-use-internal-linkage) +{ + j = {{"x", v.x()}, {"y", v.y()}, {"z", v.z()}}; +} +} // namespace issue_4320_eigen + +// The user's own namespace, using the (fake) Eigen type as an implementation +// detail behind a payload type that has nothing to do with vectors/arrays. +namespace issue_4320 +{ +// Publicly derives from issue_4320_eigen::vector3 but does *not* define its +// own to_json - it is only ever used as a temporary to reach the base +// class's to_json via ADL. +struct vector3_wrapper : issue_4320_eigen::vector3 +{ + using issue_4320_eigen::vector3::vector3; +}; + +struct payload +{ + double x, y, z; +}; + +inline vector3_wrapper to_eigen(const payload& p) // NOLINT(misc-use-internal-linkage) +{ + return {p.x, p.y, p.z}; +} + +inline void to_json(json& j, const payload& p) // NOLINT(misc-use-internal-linkage) +{ + // Unqualified call, passing a *derived* vector3_wrapper: relies on ADL + // finding issue_4320_eigen::to_json(json&, const vector3&) through the + // vector3 base class, via a derived-to-base conversion. Must NOT resolve + // to the library's own generic array-compatible to_json (an exact-match + // template for vector3_wrapper, since it also has begin()/end()), which + // would serialize this as [x, y, z] instead of {"x":x, "y":y, "z":z}. + to_json(j, to_eigen(p)); +} +} // namespace issue_4320 + +TEST_CASE("issue #4320 - custom base class must not leak nlohmann::detail into ADL") +{ + // Before the fix, basic_json unconditionally derived from a type living in + // nlohmann::detail (json_default_base), which made nlohmann::detail an + // associated namespace of every basic_json for ADL purposes. That leaked + // the library's internal generic-array to_json overload into unqualified + // to_json() calls made from user code, silently bypassing user-defined + // to_json overloads reached via a derived-to-base conversion. + const issue_4320::payload p{1.0, 2.0, 3.0}; + + json j; + to_json(j, p); + CHECK(j == json({{"x", 1.0}, {"y", 2.0}, {"z", 3.0}})); +} + +TEST_CASE("issue #5338 - truncated CBOR tagged binary subtype is rejected") +{ + const std::vector> truncated_tags = + { + {0xD8}, + {0xD9, 0x00}, + {0xDA, 0x00, 0x00, 0x00}, + {0xDB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00} + }; + + for (const auto& data : truncated_tags) + { + CAPTURE(data); + for (const auto tag_handler : + { + json::cbor_tag_handler_t::ignore, json::cbor_tag_handler_t::store + }) + { + CAPTURE(tag_handler); + const auto result = json::from_cbor(data, true, false, tag_handler); + CHECK(result.is_discarded()); + } + } +} + +TEST_CASE("issue #5402 - update(merge_objects=true) overwrites a primitive with an object") +{ + json t = {{"k", 1}}; + t.update(json{{"k", {{"x", 2}}}}, true); + CHECK(t == json({{"k", {{"x", 2}}}})); + + json mixed = {{"keep", {{"a", 1}}}, {"replace", 1}}; + mixed.update(json{{"keep", {{"b", 2}}}, {"replace", {{"x", 2}}}}, true); + CHECK(mixed == json({{"keep", {{"a", 1}, {"b", 2}}}, {"replace", {{"x", 2}}}})); +} + + +DOCTEST_CLANG_SUPPRESS_WARNING_POP From 0dd8ca9023c60e64529e9ffad71a887476c4147a Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Fri, 11 Sep 2026 08:34:43 +0200 Subject: [PATCH 15/18] Read MessagePack containers without recursing per nesting level (#5505) get_msgpack_array() and get_msgpack_object() read their elements by calling back into parse_msgpack_internal(), which calls them again for a nested container. The native call stack therefore grew with the nesting depth of the input, and each level costs only one byte to encode: 0x91 is a one-element array, so a few hundred thousand of them crash the process before any of the input is rejected (#5104). Keep the open containers on a heap stack instead, the way parser::sax_parse_internal() has always done for JSON text. A frame records how many elements are left and whether to close with end_object() or end_array(); parse_msgpack_value() reads a single value and, for a container, only opens it; and parse_msgpack_internal() loops, resuming the innermost container after each element and closing it when its count runs out. Whether the value that was begun is complete is answered by the stack being empty, so no separate bookkeeping is needed. The switch that decodes a value is untouched apart from the six container cases, which now call enter_container() rather than a reader that loops. That keeps this diff to the control flow and leaves the decoding of every other type byte-identical. enter_container() is the only place a binary reader emits start_object() or start_array(), so a check that rejects a container can be added there once and is guaranteed to run before the start event. The frame type and the stack are shared, ready for the other three formats. Verified against develop over empty, nested, counted (array 16/32, map 16/32) and truncated inputs: identical values, error codes, messages and byte offsets. 300,000 levels now report parse_error.110 instead of crashing, and a well-formed 300,000-level value is read to completion through the SAX interface, where develop crashes. Reading such a value into a basic_json needs the return-by-move change as well, without which the recursive copy constructor overflows on the way out; that is the parent commit, and the test for the value path covers the two together. Timing is unchanged: parsing 60,000 small objects and one array of a million integers is within run-to-run noise of develop either way. Signed-off-by: Niels Lohmann --- .../nlohmann/detail/input/binary_reader.hpp | 181 +++++++++++++----- single_include/nlohmann/json.hpp | 181 +++++++++++++----- tests/src/unit-msgpack.cpp | 61 ++++++ 3 files changed, 333 insertions(+), 90 deletions(-) diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index f699de1b9..a0fb10be8 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -130,6 +130,7 @@ class binary_reader const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error) { sax = sax_; + container_stack.clear(); bool result = false; switch (format) @@ -179,6 +180,69 @@ class binary_reader } private: + //////////////////////// + // nested containers // + //////////////////////// + + /*! + @brief a container that has been opened and not closed yet + + The binary readers do not call themselves once per nesting level. Like + @ref parser::sax_parse_internal, which does the same for JSON text, they + keep the containers they are inside of on a heap-allocated stack, so that + the native call stack does not grow with the nesting depth of the input + and a deeply nested value is bounded by memory rather than by the stack + (see #5104). + + The members are ordered by decreasing alignment, which is the ordering that + keeps a struct from growing as members are added to it. + */ + struct container_frame + { + container_frame(const std::size_t remaining_, const bool is_object_) noexcept + : remaining(remaining_), is_object(is_object_) {} + + /// number of elements that have not been read yet + std::size_t remaining; + /// whether to close this container with end_object() or end_array() + bool is_object; + }; + + /*! + @brief open a nested array or object + + Emits the SAX start event and records the container. This is the only + place the binary readers start a container, so a check that rejects one + can be made here and is then guaranteed to run before the start event. + + @param[in] is_object whether an object (true) or an array (false) begins + @param[in] len number of elements the container declares + + @return whether the SAX parser accepted the start event + */ + bool enter_container(const bool is_object, const std::size_t len) + { + if (JSON_HEDLEY_UNLIKELY(is_object ? !sax->start_object(len) : !sax->start_array(len))) + { + return false; + } + + container_stack.emplace_back(len, is_object); + return true; + } + + /// @copydoc enter_container + bool enter_array(const std::size_t len) + { + return enter_container(/*is_object*/false, len); + } + + /// @copydoc enter_container + bool enter_object(const std::size_t len) + { + return enter_container(/*is_object*/true, len); + } + ////////// // BSON // ////////// @@ -1422,7 +1486,17 @@ class binary_reader /*! @return whether a valid MessagePack value was passed to the SAX parser */ - bool parse_msgpack_internal() + /*! + @brief read one MessagePack value + + Reads a single value and passes it to the SAX parser. A value that begins + a container is not read to its end: the container is opened with + @ref enter_container and its elements are read by + @ref parse_msgpack_internal, so that nesting does not consume native stack. + + @return whether reading the value succeeded + */ + bool parse_msgpack_value() { switch (get()) { @@ -1578,7 +1652,7 @@ class binary_reader case 0x8D: case 0x8E: case 0x8F: - return get_msgpack_object(conditional_static_cast(static_cast(current) & 0x0Fu)); + return enter_object(conditional_static_cast(static_cast(current) & 0x0Fu)); // fixarray case 0x90: @@ -1597,7 +1671,7 @@ class binary_reader case 0x9D: case 0x9E: case 0x9F: - return get_msgpack_array(conditional_static_cast(static_cast(current) & 0x0Fu)); + return enter_array(conditional_static_cast(static_cast(current) & 0x0Fu)); // fixstr case 0xA0: @@ -1728,25 +1802,25 @@ class binary_reader case 0xDC: // array 16 { std::uint16_t len{}; - return get_number(input_format_t::msgpack, len) && get_msgpack_array(static_cast(len)); + return get_number(input_format_t::msgpack, len) && enter_array(static_cast(len)); } case 0xDD: // array 32 { std::uint32_t len{}; - return get_number(input_format_t::msgpack, len) && get_msgpack_array(conditional_static_cast(len)); + return get_number(input_format_t::msgpack, len) && enter_array(conditional_static_cast(len)); } case 0xDE: // map 16 { std::uint16_t len{}; - return get_number(input_format_t::msgpack, len) && get_msgpack_object(static_cast(len)); + return get_number(input_format_t::msgpack, len) && enter_object(static_cast(len)); } case 0xDF: // map 32 { std::uint32_t len{}; - return get_number(input_format_t::msgpack, len) && get_msgpack_object(conditional_static_cast(len)); + return get_number(input_format_t::msgpack, len) && enter_object(conditional_static_cast(len)); } // negative fixint @@ -1994,55 +2068,69 @@ class binary_reader } /*! - @param[in] len the length of the array - @return whether array creation completed + @brief read a MessagePack value and everything nested inside it + + Reads values until the one that was begun here is complete, resuming the + enclosing container each time an element ends, so that the nesting depth + of the input costs heap rather than native stack (see #5104). + + @return whether reading the value succeeded */ - bool get_msgpack_array(const std::size_t len) + bool parse_msgpack_internal() { - if (JSON_HEDLEY_UNLIKELY(!sax->start_array(len))) - { - return false; - } - - for (std::size_t i = 0; i < len; ++i) - { - if (JSON_HEDLEY_UNLIKELY(!parse_msgpack_internal())) - { - return false; - } - } - - return sax->end_array(); - } - - /*! - @param[in] len the length of the object - @return whether object creation completed - */ - bool get_msgpack_object(const std::size_t len) - { - if (JSON_HEDLEY_UNLIKELY(!sax->start_object(len))) - { - return false; - } - + // the key currently being read; hoisted out of the loop so that its + // capacity is reused across elements and across nesting levels string_t key; - for (std::size_t i = 0; i < len; ++i) + + while (true) { - get(); - if (JSON_HEDLEY_UNLIKELY(!get_msgpack_string(key) || !sax->key(key))) + if (!container_stack.empty()) + { + // copied out before anything can push onto the stack and + // invalidate a reference into it + const bool is_object = container_stack.back().is_object; + + if (container_stack.back().remaining == 0) + { + container_stack.pop_back(); + if (JSON_HEDLEY_UNLIKELY(is_object ? !sax->end_object() : !sax->end_array())) + { + return false; + } + // the value begun here is complete once its container is + if (container_stack.empty()) + { + return true; + } + continue; + } + + // claim the element about to be read + --container_stack.back().remaining; + + if (is_object) + { + get(); + key.clear(); + if (JSON_HEDLEY_UNLIKELY(!get_msgpack_string(key) || !sax->key(key))) + { + return false; + } + } + } + + if (JSON_HEDLEY_UNLIKELY(!parse_msgpack_value())) { return false; } - if (JSON_HEDLEY_UNLIKELY(!parse_msgpack_internal())) + // a value that opened a container left it on the stack; one that + // did not, and that was not inside a container, was the whole value + if (container_stack.empty()) { - return false; + return true; } - key.clear(); } - - return sax->end_object(); } //////////// @@ -3347,6 +3435,9 @@ class binary_reader /// the SAX parser json_sax_t* sax = nullptr; + /// the containers that have been opened and not closed yet; see @ref container_frame + std::vector container_stack{}; + // excluded markers in bjdata optimized type #define JSON_BINARY_READER_MAKE_BJD_OPTIMIZED_TYPE_MARKERS_ \ make_array('F', 'H', 'N', 'S', 'T', 'Z', '[', '{') diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index e8899af56..bd5e502a7 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -12079,6 +12079,7 @@ class binary_reader const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error) { sax = sax_; + container_stack.clear(); bool result = false; switch (format) @@ -12128,6 +12129,69 @@ class binary_reader } private: + //////////////////////// + // nested containers // + //////////////////////// + + /*! + @brief a container that has been opened and not closed yet + + The binary readers do not call themselves once per nesting level. Like + @ref parser::sax_parse_internal, which does the same for JSON text, they + keep the containers they are inside of on a heap-allocated stack, so that + the native call stack does not grow with the nesting depth of the input + and a deeply nested value is bounded by memory rather than by the stack + (see #5104). + + The members are ordered by decreasing alignment, which is the ordering that + keeps a struct from growing as members are added to it. + */ + struct container_frame + { + container_frame(const std::size_t remaining_, const bool is_object_) noexcept + : remaining(remaining_), is_object(is_object_) {} + + /// number of elements that have not been read yet + std::size_t remaining; + /// whether to close this container with end_object() or end_array() + bool is_object; + }; + + /*! + @brief open a nested array or object + + Emits the SAX start event and records the container. This is the only + place the binary readers start a container, so a check that rejects one + can be made here and is then guaranteed to run before the start event. + + @param[in] is_object whether an object (true) or an array (false) begins + @param[in] len number of elements the container declares + + @return whether the SAX parser accepted the start event + */ + bool enter_container(const bool is_object, const std::size_t len) + { + if (JSON_HEDLEY_UNLIKELY(is_object ? !sax->start_object(len) : !sax->start_array(len))) + { + return false; + } + + container_stack.emplace_back(len, is_object); + return true; + } + + /// @copydoc enter_container + bool enter_array(const std::size_t len) + { + return enter_container(/*is_object*/false, len); + } + + /// @copydoc enter_container + bool enter_object(const std::size_t len) + { + return enter_container(/*is_object*/true, len); + } + ////////// // BSON // ////////// @@ -13371,7 +13435,17 @@ class binary_reader /*! @return whether a valid MessagePack value was passed to the SAX parser */ - bool parse_msgpack_internal() + /*! + @brief read one MessagePack value + + Reads a single value and passes it to the SAX parser. A value that begins + a container is not read to its end: the container is opened with + @ref enter_container and its elements are read by + @ref parse_msgpack_internal, so that nesting does not consume native stack. + + @return whether reading the value succeeded + */ + bool parse_msgpack_value() { switch (get()) { @@ -13527,7 +13601,7 @@ class binary_reader case 0x8D: case 0x8E: case 0x8F: - return get_msgpack_object(conditional_static_cast(static_cast(current) & 0x0Fu)); + return enter_object(conditional_static_cast(static_cast(current) & 0x0Fu)); // fixarray case 0x90: @@ -13546,7 +13620,7 @@ class binary_reader case 0x9D: case 0x9E: case 0x9F: - return get_msgpack_array(conditional_static_cast(static_cast(current) & 0x0Fu)); + return enter_array(conditional_static_cast(static_cast(current) & 0x0Fu)); // fixstr case 0xA0: @@ -13677,25 +13751,25 @@ class binary_reader case 0xDC: // array 16 { std::uint16_t len{}; - return get_number(input_format_t::msgpack, len) && get_msgpack_array(static_cast(len)); + return get_number(input_format_t::msgpack, len) && enter_array(static_cast(len)); } case 0xDD: // array 32 { std::uint32_t len{}; - return get_number(input_format_t::msgpack, len) && get_msgpack_array(conditional_static_cast(len)); + return get_number(input_format_t::msgpack, len) && enter_array(conditional_static_cast(len)); } case 0xDE: // map 16 { std::uint16_t len{}; - return get_number(input_format_t::msgpack, len) && get_msgpack_object(static_cast(len)); + return get_number(input_format_t::msgpack, len) && enter_object(static_cast(len)); } case 0xDF: // map 32 { std::uint32_t len{}; - return get_number(input_format_t::msgpack, len) && get_msgpack_object(conditional_static_cast(len)); + return get_number(input_format_t::msgpack, len) && enter_object(conditional_static_cast(len)); } // negative fixint @@ -13943,55 +14017,69 @@ class binary_reader } /*! - @param[in] len the length of the array - @return whether array creation completed + @brief read a MessagePack value and everything nested inside it + + Reads values until the one that was begun here is complete, resuming the + enclosing container each time an element ends, so that the nesting depth + of the input costs heap rather than native stack (see #5104). + + @return whether reading the value succeeded */ - bool get_msgpack_array(const std::size_t len) + bool parse_msgpack_internal() { - if (JSON_HEDLEY_UNLIKELY(!sax->start_array(len))) - { - return false; - } - - for (std::size_t i = 0; i < len; ++i) - { - if (JSON_HEDLEY_UNLIKELY(!parse_msgpack_internal())) - { - return false; - } - } - - return sax->end_array(); - } - - /*! - @param[in] len the length of the object - @return whether object creation completed - */ - bool get_msgpack_object(const std::size_t len) - { - if (JSON_HEDLEY_UNLIKELY(!sax->start_object(len))) - { - return false; - } - + // the key currently being read; hoisted out of the loop so that its + // capacity is reused across elements and across nesting levels string_t key; - for (std::size_t i = 0; i < len; ++i) + + while (true) { - get(); - if (JSON_HEDLEY_UNLIKELY(!get_msgpack_string(key) || !sax->key(key))) + if (!container_stack.empty()) + { + // copied out before anything can push onto the stack and + // invalidate a reference into it + const bool is_object = container_stack.back().is_object; + + if (container_stack.back().remaining == 0) + { + container_stack.pop_back(); + if (JSON_HEDLEY_UNLIKELY(is_object ? !sax->end_object() : !sax->end_array())) + { + return false; + } + // the value begun here is complete once its container is + if (container_stack.empty()) + { + return true; + } + continue; + } + + // claim the element about to be read + --container_stack.back().remaining; + + if (is_object) + { + get(); + key.clear(); + if (JSON_HEDLEY_UNLIKELY(!get_msgpack_string(key) || !sax->key(key))) + { + return false; + } + } + } + + if (JSON_HEDLEY_UNLIKELY(!parse_msgpack_value())) { return false; } - if (JSON_HEDLEY_UNLIKELY(!parse_msgpack_internal())) + // a value that opened a container left it on the stack; one that + // did not, and that was not inside a container, was the whole value + if (container_stack.empty()) { - return false; + return true; } - key.clear(); } - - return sax->end_object(); } //////////// @@ -15296,6 +15384,9 @@ class binary_reader /// the SAX parser json_sax_t* sax = nullptr; + /// the containers that have been opened and not closed yet; see @ref container_frame + std::vector container_stack{}; + // excluded markers in bjdata optimized type #define JSON_BINARY_READER_MAKE_BJD_OPTIMIZED_TYPE_MARKERS_ \ make_array('F', 'H', 'N', 'S', 'T', 'Z', '[', '{') diff --git a/tests/src/unit-msgpack.cpp b/tests/src/unit-msgpack.cpp index 4358cd464..75c2ae464 100644 --- a/tests/src/unit-msgpack.cpp +++ b/tests/src/unit-msgpack.cpp @@ -1598,6 +1598,67 @@ TEST_CASE("MessagePack") } // use this testcase outside [hide] to run it with Valgrind +TEST_CASE("MessagePack nesting does not consume the call stack") +{ + // Reading a container used to call back into the value reader once per + // element, so the native call stack grew with the nesting depth of the + // input: one frame per byte for repeated 0x91 (a one-element array), which + // crashes the process long before the input is exhausted (#5104). The + // containers are kept on a heap stack now. + // + // Note that deeply nested values must not be compared, copied or dumped + // here: those operations are still recursive, and would reintroduce the + // very crash this checks for. Depth is measured by descending instead. + + SECTION("an unterminated chain is reported, not crashed on") + { + json _; + const std::vector input(300000, 0x91); + CHECK_THROWS_WITH_AS(_ = json::from_msgpack(input), "[json.exception.parse_error.110] parse error at byte 300001: syntax error while parsing MessagePack value: unexpected end of input", json::parse_error&); + CHECK(json::from_msgpack(input, true, false).is_discarded()); + } + + SECTION("a well-formed deep value is read through the SAX interface") + { + std::vector input(300000, 0x91); + input.push_back(0x01); // innermost value + + SaxCountdown accept_all(600001); + CHECK(json::sax_parse(input, &accept_all, json::input_format_t::msgpack)); + } + + SECTION("a well-formed deep value is read into a value") + { + const std::size_t depth = 10000; + std::vector input(depth, 0x91); + input.push_back(0x01); + + json j = json::from_msgpack(input); + + std::size_t measured = 0; + const json* p = &j; + while (p->is_array() && !p->empty()) + { + p = &p->front(); + ++measured; + } + CHECK(measured == depth); + CHECK(p->is_number()); + } + + SECTION("containers are still read the same way") + { + CHECK(json::from_msgpack(std::vector({0x90})) == json::array()); + CHECK(json::from_msgpack(std::vector({0x80})) == json::object()); + CHECK(json::from_msgpack(std::vector({0x92, 0x90, 0x80})) == json({json::array(), json::object()})); + CHECK(json::from_msgpack(std::vector({0x91, 0x91, 0x91, 0x90})) == json({{{json::array()}}})); + CHECK(json::from_msgpack(std::vector({0x81, 0xA1, 'a', 0x81, 0xA1, 'b', 0x92, 0x01, 0x02})) == json({{"a", {{"b", {1, 2}}}}})); + // array 16 and map 32, i.e. the counted forms + CHECK(json::from_msgpack(std::vector({0xDC, 0x00, 0x02, 0x01, 0x02})) == json({1, 2})); + CHECK(json::from_msgpack(std::vector({0xDF, 0x00, 0x00, 0x00, 0x01, 0xA1, 'k', 0xC3})) == json({{"k", true}})); + } +} + TEST_CASE("single MessagePack roundtrip") { SECTION("sample.json") From 91ab3e81f5e6fee33c03884a94a62a2b494ece6a Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Fri, 11 Sep 2026 08:34:44 +0200 Subject: [PATCH 16/18] Read CBOR containers and tags without recursing per nesting level (#5506) * Read CBOR containers and tags without recursing per nesting level get_cbor_array() and get_cbor_object() read their elements by calling back into the value reader, which called them again for a nested container, and a tag was handled by reading the tagged value the same way. All three cost native stack, and all three cost a single byte to encode: 0x9F opens an indefinite-length array, 0x81 a one-element array, and 0xC2 is a tag. Half a million of any of them crashes the process before the input runs out (#5104). Apply the shape the MessagePack reader already uses: the open containers live on the heap stack, parse_cbor_value() reads a single value and only opens a container rather than reading it to its end, and parse_cbor_internal() loops, resuming the innermost container after each element. Two things are specific to CBOR. An indefinite-length container ends at a break marker rather than at a count, and testing for that marker consumes a byte which is the first byte of the next element when it is not one; the frame's count is npos for those, and the driver tracks whether the next value starts at a fresh byte. And a tag is not a value of its own: instead of reading the tagged value by recursing, the value reader reports that a tag was read and the driver reads on, so a chain of tags costs no stack at all. The switch that decodes a value is unchanged apart from the twelve container cases and the two tag sites. Verified against the previous commit over definite and indefinite arrays and maps, all four counted forms, empty containers, nesting of the forms inside each other, truncated inputs, and all three tag handlers: identical values, error codes, messages and byte offsets. 500,000 levels of each of the three vectors now report parse_error.110 instead of crashing, and a well-formed 200,000-level value is read to completion. On performance: the driver does per element what a counted loop used to do per container, and CBOR pays for it more than MessagePack because the value reader also has to be told whether to fetch a byte. Parsing 60,000 small objects and one array of a million integers is 3 to 4 % slower than the recursive reader, measured over five alternating runs. Against develop the same two inputs are about 44 % faster, because the entry point no longer copies the value it parsed; the earlier commit in this series is what pays for that. Signed-off-by: Niels Lohmann * Make parse_cbor_internal's top a copy so it survives pop_back() top aliased container_stack.back(), and was still read (top.is_object) right after container_stack.pop_back() destroyed the element it aliased. Nothing currently reorders those two lines, but the comment claiming the reference's lifetime was already fine only accounted for reallocation from a push, not this. A trivially-copyable container_frame makes top a copy instead, so reads of it stay valid regardless of what happens to the stack; the one place that mutates the live entry now does so through container_stack.back() directly rather than through top. Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann --- .../nlohmann/detail/input/binary_reader.hpp | 193 ++++++++++-------- single_include/nlohmann/json.hpp | 193 ++++++++++-------- tests/src/unit-cbor.cpp | 87 ++++++++ 3 files changed, 303 insertions(+), 170 deletions(-) diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index a0fb10be8..74e15e1c7 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -575,9 +575,12 @@ class binary_reader @return whether a valid CBOR value was passed to the SAX parser */ - bool parse_cbor_internal(const bool get_char, - const cbor_tag_handler_t tag_handler) + bool parse_cbor_value(const bool get_char, + const cbor_tag_handler_t tag_handler, + bool& tag_pending) { + tag_pending = false; + switch (get_char ? get() : current) { // EOF @@ -769,37 +772,36 @@ class binary_reader case 0x95: case 0x96: case 0x97: - return get_cbor_array( - conditional_static_cast(static_cast(current) & 0x1Fu), tag_handler); + return enter_array(conditional_static_cast(static_cast(current) & 0x1Fu)); case 0x98: // array (one-byte uint8_t for n follows) { std::uint8_t len{}; - return get_number(input_format_t::cbor, len) && get_cbor_array(static_cast(len), tag_handler); + return get_number(input_format_t::cbor, len) && enter_array(static_cast(len)); } case 0x99: // array (two-byte uint16_t for n follow) { std::uint16_t len{}; - return get_number(input_format_t::cbor, len) && get_cbor_array(static_cast(len), tag_handler); + return get_number(input_format_t::cbor, len) && enter_array(static_cast(len)); } case 0x9A: // array (four-byte uint32_t for n follow) { std::uint32_t len{}; std::size_t size{}; - return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "array") && get_cbor_array(size, tag_handler); + return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "array") && enter_array(size); } case 0x9B: // array (eight-byte uint64_t for n follow) { std::uint64_t len{}; std::size_t size{}; - return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "array") && get_cbor_array(size, tag_handler); + return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "array") && enter_array(size); } case 0x9F: // array (indefinite length) - return get_cbor_array(detail::unknown_size(), tag_handler); + return enter_array(detail::unknown_size()); // map (0x00..0x17 pairs of data items follow) case 0xA0: @@ -826,36 +828,36 @@ class binary_reader case 0xB5: case 0xB6: case 0xB7: - return get_cbor_object(conditional_static_cast(static_cast(current) & 0x1Fu), tag_handler); + return enter_object(conditional_static_cast(static_cast(current) & 0x1Fu)); case 0xB8: // map (one-byte uint8_t for n follows) { std::uint8_t len{}; - return get_number(input_format_t::cbor, len) && get_cbor_object(static_cast(len), tag_handler); + return get_number(input_format_t::cbor, len) && enter_object(static_cast(len)); } case 0xB9: // map (two-byte uint16_t for n follow) { std::uint16_t len{}; - return get_number(input_format_t::cbor, len) && get_cbor_object(static_cast(len), tag_handler); + return get_number(input_format_t::cbor, len) && enter_object(static_cast(len)); } case 0xBA: // map (four-byte uint32_t for n follow) { std::uint32_t len{}; std::size_t size{}; - return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "map") && get_cbor_object(size, tag_handler); + return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "map") && enter_object(size); } case 0xBB: // map (eight-byte uint64_t for n follow) { std::uint64_t len{}; std::size_t size{}; - return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "map") && get_cbor_object(size, tag_handler); + return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "map") && enter_object(size); } case 0xBF: // map (indefinite length) - return get_cbor_object(detail::unknown_size(), tag_handler); + return enter_object(detail::unknown_size()); case 0xC0: // tagged item case 0xC1: @@ -939,7 +941,10 @@ class binary_reader default: break; } - return parse_cbor_internal(true, tag_handler); + // the tagged value follows; it is read by the loop in + // parse_cbor_internal() rather than by recursing here + tag_pending = true; + return true; } case cbor_tag_handler_t::store: @@ -989,7 +994,11 @@ class binary_reader break; } default: - return parse_cbor_internal(true, tag_handler); + { + // as above, the tagged value is read by the caller + tag_pending = true; + return true; + } } get(); return get_cbor_binary(b) && sax->binary(b); @@ -1387,96 +1396,110 @@ class binary_reader } /*! - @param[in] len the length of the array or detail::unknown_size() for an - array of indefinite size + @brief read a CBOR value and everything nested inside it + + Reads values until the one that was begun here is complete, resuming the + enclosing container after each element, so that the nesting depth of the + input costs heap rather than native stack (see #5104). + + @param[in] get_char whether a new character should be retrieved from the + input (true) or whether the last read character + @a current should be considered instead @param[in] tag_handler how CBOR tags should be treated - @return whether array creation completed + + @return whether reading the value succeeded */ - bool get_cbor_array(const std::size_t len, - const cbor_tag_handler_t tag_handler) + bool parse_cbor_internal(const bool get_char, + const cbor_tag_handler_t tag_handler) { - if (JSON_HEDLEY_UNLIKELY(!sax->start_array(len))) - { - return false; - } + // whether the next value starts at a fresh byte or at the one already + // read into `current` + bool fetch = get_char; - if (len != detail::unknown_size()) + // the key currently being read; hoisted out of the loop so that its + // capacity is reused across elements and across nesting levels + string_t key; + + while (true) { - for (std::size_t i = 0; i < len; ++i) + if (!container_stack.empty()) { - if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(true, tag_handler))) + // a copy, not a reference: it must stay valid across the + // pop_back() below, which destroys the container_stack element + // it would otherwise alias + container_frame top = container_stack.back(); + bool at_end = false; + + if (top.remaining != npos) { - return false; + // definite length: the container ends once its elements + // have been read + at_end = (top.remaining == 0); + if (!at_end) + { + // claim the element about to be read + --container_stack.back().remaining; + if (top.is_object) + { + get(); + } + } + fetch = true; } - } - } - else - { - while (get() != 0xFF) - { - if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(false, tag_handler))) + else { - return false; + // indefinite length: the container ends at a break marker. + // Testing for it consumes a byte, which is the first byte + // of the next element when it is not one. + at_end = (get() == 0xFF); + fetch = top.is_object; } - } - } - return sax->end_array(); - } - - /*! - @param[in] len the length of the object or detail::unknown_size() for an - object of indefinite size - @param[in] tag_handler how CBOR tags should be treated - @return whether object creation completed - */ - bool get_cbor_object(const std::size_t len, - const cbor_tag_handler_t tag_handler) - { - if (JSON_HEDLEY_UNLIKELY(!sax->start_object(len))) - { - return false; - } - - if (len != 0) - { - string_t key; - if (len != detail::unknown_size()) - { - for (std::size_t i = 0; i < len; ++i) + if (at_end) { - get(); + container_stack.pop_back(); + if (JSON_HEDLEY_UNLIKELY(top.is_object ? !sax->end_object() : !sax->end_array())) + { + return false; + } + // the value begun here is complete once its container is + if (container_stack.empty()) + { + return true; + } + continue; + } + + if (top.is_object) + { + key.clear(); if (JSON_HEDLEY_UNLIKELY(!get_cbor_string(key) || !sax->key(key))) { return false; } - - if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(true, tag_handler))) - { - return false; - } - key.clear(); + fetch = true; } } - else - { - while (get() != 0xFF) - { - if (JSON_HEDLEY_UNLIKELY(!get_cbor_string(key) || !sax->key(key))) - { - return false; - } - if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(true, tag_handler))) - { - return false; - } - key.clear(); + // a tag is not a value of its own: read on until the tagged value + bool tag_pending = false; + do + { + if (JSON_HEDLEY_UNLIKELY(!parse_cbor_value(fetch, tag_handler, tag_pending))) + { + return false; } + fetch = true; + } + while (tag_pending); + + // a value that opened a container left it on the stack; one that + // did not, and that was not inside a container, was the whole value + if (container_stack.empty()) + { + return true; } } - - return sax->end_object(); } ///////////// diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index bd5e502a7..7ba9b9b60 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -12524,9 +12524,12 @@ class binary_reader @return whether a valid CBOR value was passed to the SAX parser */ - bool parse_cbor_internal(const bool get_char, - const cbor_tag_handler_t tag_handler) + bool parse_cbor_value(const bool get_char, + const cbor_tag_handler_t tag_handler, + bool& tag_pending) { + tag_pending = false; + switch (get_char ? get() : current) { // EOF @@ -12718,37 +12721,36 @@ class binary_reader case 0x95: case 0x96: case 0x97: - return get_cbor_array( - conditional_static_cast(static_cast(current) & 0x1Fu), tag_handler); + return enter_array(conditional_static_cast(static_cast(current) & 0x1Fu)); case 0x98: // array (one-byte uint8_t for n follows) { std::uint8_t len{}; - return get_number(input_format_t::cbor, len) && get_cbor_array(static_cast(len), tag_handler); + return get_number(input_format_t::cbor, len) && enter_array(static_cast(len)); } case 0x99: // array (two-byte uint16_t for n follow) { std::uint16_t len{}; - return get_number(input_format_t::cbor, len) && get_cbor_array(static_cast(len), tag_handler); + return get_number(input_format_t::cbor, len) && enter_array(static_cast(len)); } case 0x9A: // array (four-byte uint32_t for n follow) { std::uint32_t len{}; std::size_t size{}; - return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "array") && get_cbor_array(size, tag_handler); + return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "array") && enter_array(size); } case 0x9B: // array (eight-byte uint64_t for n follow) { std::uint64_t len{}; std::size_t size{}; - return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "array") && get_cbor_array(size, tag_handler); + return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "array") && enter_array(size); } case 0x9F: // array (indefinite length) - return get_cbor_array(detail::unknown_size(), tag_handler); + return enter_array(detail::unknown_size()); // map (0x00..0x17 pairs of data items follow) case 0xA0: @@ -12775,36 +12777,36 @@ class binary_reader case 0xB5: case 0xB6: case 0xB7: - return get_cbor_object(conditional_static_cast(static_cast(current) & 0x1Fu), tag_handler); + return enter_object(conditional_static_cast(static_cast(current) & 0x1Fu)); case 0xB8: // map (one-byte uint8_t for n follows) { std::uint8_t len{}; - return get_number(input_format_t::cbor, len) && get_cbor_object(static_cast(len), tag_handler); + return get_number(input_format_t::cbor, len) && enter_object(static_cast(len)); } case 0xB9: // map (two-byte uint16_t for n follow) { std::uint16_t len{}; - return get_number(input_format_t::cbor, len) && get_cbor_object(static_cast(len), tag_handler); + return get_number(input_format_t::cbor, len) && enter_object(static_cast(len)); } case 0xBA: // map (four-byte uint32_t for n follow) { std::uint32_t len{}; std::size_t size{}; - return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "map") && get_cbor_object(size, tag_handler); + return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "map") && enter_object(size); } case 0xBB: // map (eight-byte uint64_t for n follow) { std::uint64_t len{}; std::size_t size{}; - return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "map") && get_cbor_object(size, tag_handler); + return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "map") && enter_object(size); } case 0xBF: // map (indefinite length) - return get_cbor_object(detail::unknown_size(), tag_handler); + return enter_object(detail::unknown_size()); case 0xC0: // tagged item case 0xC1: @@ -12888,7 +12890,10 @@ class binary_reader default: break; } - return parse_cbor_internal(true, tag_handler); + // the tagged value follows; it is read by the loop in + // parse_cbor_internal() rather than by recursing here + tag_pending = true; + return true; } case cbor_tag_handler_t::store: @@ -12938,7 +12943,11 @@ class binary_reader break; } default: - return parse_cbor_internal(true, tag_handler); + { + // as above, the tagged value is read by the caller + tag_pending = true; + return true; + } } get(); return get_cbor_binary(b) && sax->binary(b); @@ -13336,96 +13345,110 @@ class binary_reader } /*! - @param[in] len the length of the array or detail::unknown_size() for an - array of indefinite size + @brief read a CBOR value and everything nested inside it + + Reads values until the one that was begun here is complete, resuming the + enclosing container after each element, so that the nesting depth of the + input costs heap rather than native stack (see #5104). + + @param[in] get_char whether a new character should be retrieved from the + input (true) or whether the last read character + @a current should be considered instead @param[in] tag_handler how CBOR tags should be treated - @return whether array creation completed + + @return whether reading the value succeeded */ - bool get_cbor_array(const std::size_t len, - const cbor_tag_handler_t tag_handler) + bool parse_cbor_internal(const bool get_char, + const cbor_tag_handler_t tag_handler) { - if (JSON_HEDLEY_UNLIKELY(!sax->start_array(len))) - { - return false; - } + // whether the next value starts at a fresh byte or at the one already + // read into `current` + bool fetch = get_char; - if (len != detail::unknown_size()) + // the key currently being read; hoisted out of the loop so that its + // capacity is reused across elements and across nesting levels + string_t key; + + while (true) { - for (std::size_t i = 0; i < len; ++i) + if (!container_stack.empty()) { - if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(true, tag_handler))) + // a copy, not a reference: it must stay valid across the + // pop_back() below, which destroys the container_stack element + // it would otherwise alias + container_frame top = container_stack.back(); + bool at_end = false; + + if (top.remaining != npos) { - return false; + // definite length: the container ends once its elements + // have been read + at_end = (top.remaining == 0); + if (!at_end) + { + // claim the element about to be read + --container_stack.back().remaining; + if (top.is_object) + { + get(); + } + } + fetch = true; } - } - } - else - { - while (get() != 0xFF) - { - if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(false, tag_handler))) + else { - return false; + // indefinite length: the container ends at a break marker. + // Testing for it consumes a byte, which is the first byte + // of the next element when it is not one. + at_end = (get() == 0xFF); + fetch = top.is_object; } - } - } - return sax->end_array(); - } - - /*! - @param[in] len the length of the object or detail::unknown_size() for an - object of indefinite size - @param[in] tag_handler how CBOR tags should be treated - @return whether object creation completed - */ - bool get_cbor_object(const std::size_t len, - const cbor_tag_handler_t tag_handler) - { - if (JSON_HEDLEY_UNLIKELY(!sax->start_object(len))) - { - return false; - } - - if (len != 0) - { - string_t key; - if (len != detail::unknown_size()) - { - for (std::size_t i = 0; i < len; ++i) + if (at_end) { - get(); + container_stack.pop_back(); + if (JSON_HEDLEY_UNLIKELY(top.is_object ? !sax->end_object() : !sax->end_array())) + { + return false; + } + // the value begun here is complete once its container is + if (container_stack.empty()) + { + return true; + } + continue; + } + + if (top.is_object) + { + key.clear(); if (JSON_HEDLEY_UNLIKELY(!get_cbor_string(key) || !sax->key(key))) { return false; } - - if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(true, tag_handler))) - { - return false; - } - key.clear(); + fetch = true; } } - else - { - while (get() != 0xFF) - { - if (JSON_HEDLEY_UNLIKELY(!get_cbor_string(key) || !sax->key(key))) - { - return false; - } - if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(true, tag_handler))) - { - return false; - } - key.clear(); + // a tag is not a value of its own: read on until the tagged value + bool tag_pending = false; + do + { + if (JSON_HEDLEY_UNLIKELY(!parse_cbor_value(fetch, tag_handler, tag_pending))) + { + return false; } + fetch = true; + } + while (tag_pending); + + // a value that opened a container left it on the stack; one that + // did not, and that was not inside a container, was the whole value + if (container_stack.empty()) + { + return true; } } - - return sax->end_object(); } ///////////// diff --git a/tests/src/unit-cbor.cpp b/tests/src/unit-cbor.cpp index 96b30e142..032e6641b 100644 --- a/tests/src/unit-cbor.cpp +++ b/tests/src/unit-cbor.cpp @@ -2035,6 +2035,93 @@ TEST_CASE("CBOR definite length equal to the indefinite-length sentinel") } } +TEST_CASE("CBOR nesting does not consume the call stack") +{ + // Containers used to be read by calling back into the value reader once + // per element, and a tag by calling it for the tagged value, so the native + // call stack grew with the nesting depth of the input. Each of the three + // costs a single byte to encode -- 0x9F, 0x81 and 0xC2 -- so a payload of + // repeated bytes crashed the process (#5104). The containers are kept on a + // heap stack now, and a tag is read in a loop. + // + // Deeply nested values must not be compared, copied or dumped here: those + // operations are still recursive and would reintroduce the crash. + json _; + + SECTION("indefinite-length containers") + { + const std::vector input(500000, 0x9F); + CHECK_THROWS_WITH_AS(_ = json::from_cbor(input), "[json.exception.parse_error.110] parse error at byte 500001: syntax error while parsing CBOR value: unexpected end of input", json::parse_error&); + CHECK(json::from_cbor(input, true, false).is_discarded()); + } + + SECTION("definite-length containers") + { + const std::vector input(500000, 0x81); + CHECK_THROWS_WITH_AS(_ = json::from_cbor(input), "[json.exception.parse_error.110] parse error at byte 500001: syntax error while parsing CBOR value: unexpected end of input", json::parse_error&); + CHECK(json::from_cbor(input, true, false).is_discarded()); + } + + SECTION("tags") + { + // a tag is not a value of its own, so a chain of them used to recurse + const std::vector input(500000, 0xC2); + CHECK_THROWS_WITH_AS(_ = json::from_cbor(input, true, true, json::cbor_tag_handler_t::ignore), "[json.exception.parse_error.110] parse error at byte 500001: syntax error while parsing CBOR value: unexpected end of input", json::parse_error&); + CHECK(json::from_cbor(input, true, false, json::cbor_tag_handler_t::ignore).is_discarded()); + } + + SECTION("a well-formed deep value is read through the SAX interface") + { + std::vector input(200000, 0x9F); + input.insert(input.end(), 200000, 0xFF); + + SaxCountdown accept_all(1000000); + CHECK(json::sax_parse(input, &accept_all, json::input_format_t::cbor)); + } + + SECTION("a well-formed deep value is read into a value") + { + const std::size_t depth = 10000; + std::vector input(depth, 0x81); + input.push_back(0x00); + + json j = json::from_cbor(input); + + std::size_t measured = 0; + const json* p = &j; + while (p->is_array() && !p->empty()) + { + p = &p->front(); + ++measured; + } + CHECK(measured == depth); + CHECK(p->is_number()); + } + + SECTION("containers are still read the same way") + { + CHECK(json::from_cbor(std::vector({0x80})) == json::array()); + CHECK(json::from_cbor(std::vector({0xA0})) == json::object()); + CHECK(json::from_cbor(std::vector({0x9F, 0xFF})) == json::array()); + CHECK(json::from_cbor(std::vector({0xBF, 0xFF})) == json::object()); + CHECK(json::from_cbor(std::vector({0x9F, 0x01, 0x02, 0xFF})) == json({1, 2})); + CHECK(json::from_cbor(std::vector({0xBF, 0x61, 'a', 0x01, 0xFF})) == json({{"a", 1}})); + // definite and indefinite forms nested inside each other + CHECK(json::from_cbor(std::vector({0x9F, 0x82, 0x01, 0x02, 0xA1, 0x61, 'k', 0xBF, 0xFF, 0xFF})) == json({{1, 2}, {{"k", json::object()}}})); + } + + SECTION("tagged values are still read the same way") + { + const auto ignore = json::cbor_tag_handler_t::ignore; + CHECK(json::from_cbor(std::vector({0xC2, 0x01}), true, true, ignore) == json(1)); + // a chain of tags resolves to the value that follows it + CHECK(json::from_cbor(std::vector({0xC2, 0xC2, 0xC2, 0x01}), true, true, ignore) == json(1)); + // a tag inside a container, and one in front of a container + CHECK(json::from_cbor(std::vector({0x82, 0xC2, 0x01, 0x02}), true, true, ignore) == json({1, 2})); + CHECK(json::from_cbor(std::vector({0xC2, 0x82, 0x01, 0x02}), true, true, ignore) == json({1, 2})); + } +} + TEST_CASE("CBOR indefinite-length strings do not recurse per chunk") { // Reading an indefinite-length string or byte array used to call itself From a14619b35452421c5c35e76a2c5d454a1c09274a Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Fri, 11 Sep 2026 08:34:44 +0200 Subject: [PATCH 17/18] Read UBJSON and BJData containers without recursing per nesting level (#5507) * Read UBJSON and BJData containers without recursing per nesting level get_ubjson_array() and get_ubjson_object() read their elements by calling back into the value reader, which called them again for a nested container, so the native call stack grew with the nesting depth of the input. '[' alone opens a container, so half a million of them crashes the process before the input runs out (#5104). The optimized forms reach the same path through a size or type annotation, and in plain UBJSON '[' and '{' are permitted as the type of an optimized container, so "[$[#i\x01" repeated nests just as deeply at six bytes a level. Both readers now only open their container, and parse_ubjson_internal() loops: it closes the containers that have ended, claims the next element of the innermost one, reads its key when it is an object, and works out the marker of the value to read next. That last part is where the formats differ, and the loop follows what the four element loops used to do: - a sized, typed container gives its elements no marker of their own - a sized, untyped container reads one for each element - a container that ends at a marker has the byte already, from the test against ']' or '}'; for an object it is the first byte of the key The ND-array wrapper and the 'B' binary shortcut stay as they are. Both read a complete value rather than opening a container, and their elements are always scalars: BJData does not permit '[' or '{' as an optimized type, which is also why only plain UBJSON needed the type-marker case above. A container of no-ops keeps its behaviour of holding no elements while still announcing its declared size to the SAX parser, by opening it and then setting its count to zero. unit-ubjson and unit-bjdata pass unchanged, 1.39 million assertions between them, and a behaviour comparison against the previous commit over every container form -- sized, unsized, typed, untyped, empty, no-op, ND-array, binary, and the forms nested inside one another -- gives identical values, error codes, messages and byte offsets. 500,000 levels of each vector now report a parse error instead of crashing, and a well-formed 100,000-level value is read to completion. The driver costs about 3 % on parsing 60,000 small objects and one array of a million integers, for the reason given in the previous commit; reading the frame once per element rather than per branch halved what it cost before. Signed-off-by: Niels Lohmann * Make the UBJSON/BJData advance loop's top a copy, not a reference Same issue as the CBOR reader: top aliased container_stack.back() and was read (top.is_object) right after container_stack.pop_back() ended its lifetime. A copy stays valid regardless of what happens to the stack; the one place that mutates the live entry (--top.remaining) now goes through container_stack.back() directly. Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann --- .../nlohmann/detail/input/binary_reader.hpp | 227 ++++++++++-------- single_include/nlohmann/json.hpp | 227 ++++++++++-------- tests/src/unit-ubjson.cpp | 105 ++++++++ 3 files changed, 347 insertions(+), 212 deletions(-) diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index 74e15e1c7..4295bf229 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -199,11 +199,16 @@ class binary_reader */ struct container_frame { - container_frame(const std::size_t remaining_, const bool is_object_) noexcept - : remaining(remaining_), is_object(is_object_) {} + container_frame(const std::size_t remaining_, const bool is_object_, + const char_int_type type_marker_ = 0) noexcept + : remaining(remaining_), type_marker(type_marker_), is_object(is_object_) {} - /// number of elements that have not been read yet + /// number of elements that have not been read yet, or npos when the + /// container is not sized and ends at a marker instead std::size_t remaining; + /// UBJSON/BJData: the type marker of an optimized container, so that + /// its elements are read without one of their own; 0 otherwise + char_int_type type_marker; /// whether to close this container with end_object() or end_array() bool is_object; }; @@ -220,27 +225,28 @@ class binary_reader @return whether the SAX parser accepted the start event */ - bool enter_container(const bool is_object, const std::size_t len) + bool enter_container(const bool is_object, const std::size_t len, + const char_int_type type_marker = 0) { if (JSON_HEDLEY_UNLIKELY(is_object ? !sax->start_object(len) : !sax->start_array(len))) { return false; } - container_stack.emplace_back(len, is_object); + container_stack.emplace_back(len, is_object, type_marker); return true; } /// @copydoc enter_container - bool enter_array(const std::size_t len) + bool enter_array(const std::size_t len, const char_int_type type_marker = 0) { - return enter_container(/*is_object*/false, len); + return enter_container(/*is_object*/false, len, type_marker); } /// @copydoc enter_container - bool enter_object(const std::size_t len) + bool enter_object(const std::size_t len, const char_int_type type_marker = 0) { - return enter_container(/*is_object*/true, len); + return enter_container(/*is_object*/true, len, type_marker); } ////////// @@ -2169,7 +2175,103 @@ class binary_reader */ bool parse_ubjson_internal(const bool get_char = true) { - return get_ubjson_value(get_char ? get_ignore_noop() : current); + // the key currently being read; hoisted out of the loop so that its + // capacity is reused across elements and across nesting levels + string_t key; + + // the type marker of the value to read next + char_int_type prefix = get_char ? get_ignore_noop() : current; + + while (true) + { + const std::size_t depth = container_stack.size(); + + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(prefix))) + { + return false; + } + + // the value begun here is complete once it is not inside anything + if (container_stack.empty()) + { + return true; + } + + // a value was completed rather than a container opened; a + // container that ends at a marker needs the next byte to test + if (container_stack.size() == depth && container_stack.back().remaining == npos) + { + get_ignore_noop(); + } + + // advance to the next element, closing the containers that ended. + // top is a copy, not a reference: it must stay valid across the + // pop_back() below, which destroys the container_stack element it + // would otherwise alias. + for (;;) + { + container_frame top = container_stack.back(); + + if (top.remaining != npos) + { + if (top.remaining != 0) + { + --container_stack.back().remaining; + if (top.is_object) + { + key.clear(); + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key))) + { + return false; + } + } + // an optimized container gives its elements no marker + prefix = (top.type_marker != 0) ? top.type_marker : get_ignore_noop(); + break; + } + } + // the end marker is compared against a literal rather than + // against a conditional expression, because char_int_type is + // unsigned for some input adapters and MSVC then reports the + // comparison as a signed/unsigned mismatch + else if (top.is_object ? (current != '}') : (current != ']')) + { + // a container that ends at a marker is never optimized, so + // every element carries its own marker; for an object the + // byte tested above is the first byte of the key + if (top.is_object) + { + key.clear(); + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key, false) || !sax->key(key))) + { + return false; + } + prefix = get_ignore_noop(); + } + else + { + prefix = current; + } + break; + } + + container_stack.pop_back(); + if (JSON_HEDLEY_UNLIKELY(top.is_object ? !sax->end_object() : !sax->end_array())) + { + return false; + } + if (container_stack.empty()) + { + return true; + } + // the container that just ended was an element of the one + // below it, which may need the next byte for its own test + if (container_stack.back().remaining == npos) + { + get_ignore_noop(); + } + } + } } /*! @@ -2941,53 +3043,22 @@ class binary_reader exception_message(input_format, "excessive array size", "size"), nullptr)); } - if (JSON_HEDLEY_UNLIKELY(!sax->start_array(size_and_type.first))) + if (JSON_HEDLEY_UNLIKELY(!enter_array(size_and_type.first, size_and_type.second))) { return false; } - if (size_and_type.second != 0) + if (size_and_type.second == 'N') { - if (size_and_type.second != 'N') - { - for (std::size_t i = 0; i < size_and_type.first; ++i) - { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(size_and_type.second))) - { - return false; - } - } - } - } - else - { - for (std::size_t i = 0; i < size_and_type.first; ++i) - { - if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal())) - { - return false; - } - } - } - } - else - { - if (JSON_HEDLEY_UNLIKELY(!sax->start_array(detail::unknown_size()))) - { - return false; + // a no-op is not a value, so a container of them holds none; + // the declared size has already been passed to the SAX parser + container_stack.back().remaining = 0; } - while (current != ']') - { - if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal(false))) - { - return false; - } - get_ignore_noop(); - } + return true; } - return sax->end_array(); + return enter_array(detail::unknown_size()); } /*! @@ -3009,68 +3080,12 @@ class binary_reader exception_message(input_format, "BJData object does not support ND-array size in optimized format", "object"), nullptr)); } - string_t key; if (size_and_type.first != npos) { - if (JSON_HEDLEY_UNLIKELY(!sax->start_object(size_and_type.first))) - { - return false; - } - - if (size_and_type.second != 0) - { - for (std::size_t i = 0; i < size_and_type.first; ++i) - { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key))) - { - return false; - } - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(size_and_type.second))) - { - return false; - } - key.clear(); - } - } - else - { - for (std::size_t i = 0; i < size_and_type.first; ++i) - { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key))) - { - return false; - } - if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal())) - { - return false; - } - key.clear(); - } - } - } - else - { - if (JSON_HEDLEY_UNLIKELY(!sax->start_object(detail::unknown_size()))) - { - return false; - } - - while (current != '}') - { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key, false) || !sax->key(key))) - { - return false; - } - if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal())) - { - return false; - } - get_ignore_noop(); - key.clear(); - } + return enter_object(size_and_type.first, size_and_type.second); } - return sax->end_object(); + return enter_object(detail::unknown_size()); } // Note, no reader for UBJSON binary types is implemented because they do diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 7ba9b9b60..295a0d47f 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -12148,11 +12148,16 @@ class binary_reader */ struct container_frame { - container_frame(const std::size_t remaining_, const bool is_object_) noexcept - : remaining(remaining_), is_object(is_object_) {} + container_frame(const std::size_t remaining_, const bool is_object_, + const char_int_type type_marker_ = 0) noexcept + : remaining(remaining_), type_marker(type_marker_), is_object(is_object_) {} - /// number of elements that have not been read yet + /// number of elements that have not been read yet, or npos when the + /// container is not sized and ends at a marker instead std::size_t remaining; + /// UBJSON/BJData: the type marker of an optimized container, so that + /// its elements are read without one of their own; 0 otherwise + char_int_type type_marker; /// whether to close this container with end_object() or end_array() bool is_object; }; @@ -12169,27 +12174,28 @@ class binary_reader @return whether the SAX parser accepted the start event */ - bool enter_container(const bool is_object, const std::size_t len) + bool enter_container(const bool is_object, const std::size_t len, + const char_int_type type_marker = 0) { if (JSON_HEDLEY_UNLIKELY(is_object ? !sax->start_object(len) : !sax->start_array(len))) { return false; } - container_stack.emplace_back(len, is_object); + container_stack.emplace_back(len, is_object, type_marker); return true; } /// @copydoc enter_container - bool enter_array(const std::size_t len) + bool enter_array(const std::size_t len, const char_int_type type_marker = 0) { - return enter_container(/*is_object*/false, len); + return enter_container(/*is_object*/false, len, type_marker); } /// @copydoc enter_container - bool enter_object(const std::size_t len) + bool enter_object(const std::size_t len, const char_int_type type_marker = 0) { - return enter_container(/*is_object*/true, len); + return enter_container(/*is_object*/true, len, type_marker); } ////////// @@ -14118,7 +14124,103 @@ class binary_reader */ bool parse_ubjson_internal(const bool get_char = true) { - return get_ubjson_value(get_char ? get_ignore_noop() : current); + // the key currently being read; hoisted out of the loop so that its + // capacity is reused across elements and across nesting levels + string_t key; + + // the type marker of the value to read next + char_int_type prefix = get_char ? get_ignore_noop() : current; + + while (true) + { + const std::size_t depth = container_stack.size(); + + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(prefix))) + { + return false; + } + + // the value begun here is complete once it is not inside anything + if (container_stack.empty()) + { + return true; + } + + // a value was completed rather than a container opened; a + // container that ends at a marker needs the next byte to test + if (container_stack.size() == depth && container_stack.back().remaining == npos) + { + get_ignore_noop(); + } + + // advance to the next element, closing the containers that ended. + // top is a copy, not a reference: it must stay valid across the + // pop_back() below, which destroys the container_stack element it + // would otherwise alias. + for (;;) + { + container_frame top = container_stack.back(); + + if (top.remaining != npos) + { + if (top.remaining != 0) + { + --container_stack.back().remaining; + if (top.is_object) + { + key.clear(); + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key))) + { + return false; + } + } + // an optimized container gives its elements no marker + prefix = (top.type_marker != 0) ? top.type_marker : get_ignore_noop(); + break; + } + } + // the end marker is compared against a literal rather than + // against a conditional expression, because char_int_type is + // unsigned for some input adapters and MSVC then reports the + // comparison as a signed/unsigned mismatch + else if (top.is_object ? (current != '}') : (current != ']')) + { + // a container that ends at a marker is never optimized, so + // every element carries its own marker; for an object the + // byte tested above is the first byte of the key + if (top.is_object) + { + key.clear(); + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key, false) || !sax->key(key))) + { + return false; + } + prefix = get_ignore_noop(); + } + else + { + prefix = current; + } + break; + } + + container_stack.pop_back(); + if (JSON_HEDLEY_UNLIKELY(top.is_object ? !sax->end_object() : !sax->end_array())) + { + return false; + } + if (container_stack.empty()) + { + return true; + } + // the container that just ended was an element of the one + // below it, which may need the next byte for its own test + if (container_stack.back().remaining == npos) + { + get_ignore_noop(); + } + } + } } /*! @@ -14890,53 +14992,22 @@ class binary_reader exception_message(input_format, "excessive array size", "size"), nullptr)); } - if (JSON_HEDLEY_UNLIKELY(!sax->start_array(size_and_type.first))) + if (JSON_HEDLEY_UNLIKELY(!enter_array(size_and_type.first, size_and_type.second))) { return false; } - if (size_and_type.second != 0) + if (size_and_type.second == 'N') { - if (size_and_type.second != 'N') - { - for (std::size_t i = 0; i < size_and_type.first; ++i) - { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(size_and_type.second))) - { - return false; - } - } - } - } - else - { - for (std::size_t i = 0; i < size_and_type.first; ++i) - { - if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal())) - { - return false; - } - } - } - } - else - { - if (JSON_HEDLEY_UNLIKELY(!sax->start_array(detail::unknown_size()))) - { - return false; + // a no-op is not a value, so a container of them holds none; + // the declared size has already been passed to the SAX parser + container_stack.back().remaining = 0; } - while (current != ']') - { - if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal(false))) - { - return false; - } - get_ignore_noop(); - } + return true; } - return sax->end_array(); + return enter_array(detail::unknown_size()); } /*! @@ -14958,68 +15029,12 @@ class binary_reader exception_message(input_format, "BJData object does not support ND-array size in optimized format", "object"), nullptr)); } - string_t key; if (size_and_type.first != npos) { - if (JSON_HEDLEY_UNLIKELY(!sax->start_object(size_and_type.first))) - { - return false; - } - - if (size_and_type.second != 0) - { - for (std::size_t i = 0; i < size_and_type.first; ++i) - { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key))) - { - return false; - } - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(size_and_type.second))) - { - return false; - } - key.clear(); - } - } - else - { - for (std::size_t i = 0; i < size_and_type.first; ++i) - { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key))) - { - return false; - } - if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal())) - { - return false; - } - key.clear(); - } - } - } - else - { - if (JSON_HEDLEY_UNLIKELY(!sax->start_object(detail::unknown_size()))) - { - return false; - } - - while (current != '}') - { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key, false) || !sax->key(key))) - { - return false; - } - if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal())) - { - return false; - } - get_ignore_noop(); - key.clear(); - } + return enter_object(size_and_type.first, size_and_type.second); } - return sax->end_object(); + return enter_object(detail::unknown_size()); } // Note, no reader for UBJSON binary types is implemented because they do diff --git a/tests/src/unit-ubjson.cpp b/tests/src/unit-ubjson.cpp index 03446878f..2ebb55b90 100644 --- a/tests/src/unit-ubjson.cpp +++ b/tests/src/unit-ubjson.cpp @@ -2149,6 +2149,111 @@ TEST_CASE("UBJSON") } } +TEST_CASE("UBJSON nesting does not consume the call stack") +{ + // Containers used to be read by calling back into the value reader once + // per element, so the native call stack grew with the nesting depth of the + // input. '[' alone opens a container, so a payload of repeated '[' crashed + // the process (#5104), as did the optimized forms, which reach the same + // path through a type or size annotation. The containers are kept on a + // heap stack now. + // + // Deeply nested values must not be compared, copied or dumped here: those + // operations are still recursive and would reintroduce the crash. + json _; + + SECTION("containers that end at a marker") + { + const std::vector input(500000, '['); + CHECK_THROWS_WITH_AS(_ = json::from_ubjson(input), "[json.exception.parse_error.110] parse error at byte 500001: syntax error while parsing UBJSON value: unexpected end of input", json::parse_error&); + CHECK(json::from_ubjson(input, true, false).is_discarded()); + } + + SECTION("containers with a size") + { + std::vector input; + for (std::size_t i = 0; i < 100000; ++i) + { + input.push_back('['); + input.push_back('#'); + input.push_back('i'); + input.push_back(1); + } + CHECK_THROWS_AS(_ = json::from_ubjson(input), json::parse_error&); + CHECK(json::from_ubjson(input, true, false).is_discarded()); + } + + SECTION("containers with a type and a size") + { + // '[' is a permitted optimized type in UBJSON, so each element of such + // a container is itself a container, read without a marker of its own + std::vector input; + for (std::size_t i = 0; i < 100000; ++i) + { + const std::vector level = {'[', '$', '[', '#', 'i', 1}; + input.insert(input.end(), level.begin(), level.end()); + } + CHECK_THROWS_AS(_ = json::from_ubjson(input), json::parse_error&); + CHECK(json::from_ubjson(input, true, false).is_discarded()); + } + + SECTION("a well-formed deep value is read through the SAX interface") + { + std::vector input(100000, '['); + input.insert(input.end(), 100000, ']'); + + SaxCountdown accept_all(1000000); + CHECK(json::sax_parse(input, &accept_all, json::input_format_t::ubjson)); + } + + SECTION("a well-formed deep value is read into a value") + { + const std::size_t depth = 10000; + std::vector input(depth, '['); + input.insert(input.end(), depth, ']'); + + json j = json::from_ubjson(input); + + std::size_t measured = 0; + const json* p = &j; + while (p->is_array() && !p->empty()) + { + p = &p->front(); + ++measured; + } + // the innermost array is empty, so the descent stops one level short + CHECK(measured == depth - 1); + } + + SECTION("containers are still read the same way") + { + CHECK(json::from_ubjson(std::vector({'[', ']'})) == json::array()); + CHECK(json::from_ubjson(std::vector({'{', '}'})) == json::object()); + CHECK(json::from_ubjson(std::vector({'[', '#', 'i', 0})) == json::array()); + CHECK(json::from_ubjson(std::vector({'{', '#', 'i', 0})) == json::object()); + CHECK(json::from_ubjson(std::vector({'[', '$', 'i', '#', 'i', 2, 1, 2})) == json({1, 2})); + CHECK(json::from_ubjson(std::vector({'[', '#', 'i', 2, 'i', 1, 'i', 2})) == json({1, 2})); + CHECK(json::from_ubjson(std::vector({'{', '$', 'i', '#', 'i', 1, 'i', 1, 'a', 1})) == json({{"a", 1}})); + // a no-op is not a value, so a container of them holds none + CHECK(json::from_ubjson(std::vector({'[', '$', 'N', '#', 'i', 2})) == json::array()); + // sized and unsized forms nested inside one another + CHECK(json::from_ubjson(std::vector({'[', '[', '#', 'i', 2, 'i', 1, 'i', 2, ']'})) == json({{1, 2}})); + CHECK(json::from_ubjson(std::vector({'[', '#', 'i', 1, '[', 'i', 1, ']'})) == json({{1}})); + // an optimized container of containers + CHECK(json::from_ubjson(std::vector({'[', '$', '[', '#', 'i', 2, 'i', 1, ']', 'i', 2, ']'})) == json({{1}, {2}})); + } + + SECTION("BJData containers are still read the same way") + { + // the ND-array wrapper and the binary shortcut are complete values, + // not containers the reader descends into + CHECK(json::from_bjdata(std::vector({'[', '$', 'U', '#', '[', '$', 'i', '#', 'i', 2, 2, 3, 1, 2, 3, 4, 5, 6})) == + json({{"_ArrayType_", "uint8"}, {"_ArraySize_", {2, 3}}, {"_ArrayData_", {1, 2, 3, 4, 5, 6}}})); + CHECK(json::from_bjdata(std::vector({'[', '$', 'i', '#', 'i', 2, 1, 2})) == json({1, 2})); + CHECK(json::from_bjdata(std::vector({'[', '[', 'i', 1, ']', ']'})) == json({{1}})); + } +} + TEST_CASE("UBJSON optimized arrays of a valueless type are bounded") { // An element of type 'Z', 'T' or 'F' is encoded by its marker alone, so an From 0452641c186bdd64c245cfe57a185dcbcb7a6d30 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Fri, 11 Sep 2026 08:34:45 +0200 Subject: [PATCH 18/18] Read BSON documents without recursing per nesting level (#5508) * Read BSON documents without recursing per nesting level An embedded document (record type 0x03) or array (0x04) was read by calling back into the document reader, which read its element list, which called the element reader again for the next embedded one. The native call stack therefore grew with the nesting depth of the input, and about seven bytes buy a level, so a document of a few hundred kilobytes crashes the process (#5104). This is the last of the four binary formats to still do that. Apply the same shape as the other three: open_bson_document() reads the size prefix and opens the document, parse_bson_element_internal() calls it for both record types instead of recursing, and parse_bson_internal() loops over the element list of whichever document is innermost, closing it when its terminator is reached and resuming the one below. check_bson_document_size() is unchanged, and so is when it runs: a document is still measured from the byte before its size prefix to the byte after its terminator, and still reported before the end event. The frame carries those two values, which is what a per-document check needs once the reads are interleaved rather than nested. Nothing else about the element reader changes. unit-bson passes unchanged. Round trips through to_bson of nested objects, arrays, arrays of objects and mixed nesting are identical to the previous commit, as are the errors for a truncated document, an unsupported record type, a negative size and a size that does not match, including their byte offsets. A 30,000-level document built by to_bson is now read to completion where it used to crash. Note for sequencing: #5185 changes parse_bson_internal(), the element list and the array reader, which are the functions this commit restructures. It should land first; this commit then keeps its checks and moves them onto the loop. Signed-off-by: Niels Lohmann * Make parse_bson_internal's end-of-document top a copy, not a reference Same issue as the CBOR and UBJSON/BJData readers: top aliased container_stack.back() and was read (top.is_object) right after container_stack.pop_back() ended its lifetime. A copy stays valid regardless of what happens to the stack; nothing here mutates the live entry, so no field needs to go through container_stack.back() directly. Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann --- .../nlohmann/detail/input/binary_reader.hpp | 171 +++++++++--------- single_include/nlohmann/json.hpp | 171 +++++++++--------- tests/src/unit-bson.cpp | 85 +++++++++ 3 files changed, 257 insertions(+), 170 deletions(-) diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index 4295bf229..efa93070d 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -206,9 +206,14 @@ class binary_reader /// number of elements that have not been read yet, or npos when the /// container is not sized and ends at a marker instead std::size_t remaining; + /// BSON: value of chars_read before this document's size prefix, which + /// check_bson_document_size() needs once the document has been read + std::size_t start_position = 0; /// UBJSON/BJData: the type marker of an optimized container, so that /// its elements are read without one of their own; 0 otherwise char_int_type type_marker; + /// BSON: the size this document declares, in bytes + std::int32_t declared_size = 0; /// whether to close this container with end_object() or end_array() bool is_object; }; @@ -283,8 +288,10 @@ class binary_reader @brief Reads in a BSON-object and passes it to the SAX-parser. @return whether a valid BSON-value was passed to the SAX parser */ - bool parse_bson_internal() + bool open_bson_document(const bool is_object) { + // recorded before the size prefix is read, because + // check_bson_document_size() measures the document from here const std::size_t document_start = chars_read; std::int32_t document_size{}; if (!get_number(input_format_t::bson, document_size)) @@ -292,22 +299,91 @@ class binary_reader return false; } - if (JSON_HEDLEY_UNLIKELY(!sax->start_object(detail::unknown_size()))) + if (JSON_HEDLEY_UNLIKELY(!enter_container(is_object, detail::unknown_size()))) { return false; } - if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_list(/*is_array*/false))) + container_frame& frame = container_stack.back(); + frame.start_position = document_start; + frame.declared_size = document_size; + return true; + } + + /*! + @brief read a BSON document and everything nested inside it + + Reads elements until the document that was begun here is complete, + resuming the enclosing document each time an embedded one ends, so that + the nesting depth of the input costs heap rather than native stack + (see #5104). + + @return whether reading the document succeeded + */ + bool parse_bson_internal() + { + if (JSON_HEDLEY_UNLIKELY(!open_bson_document(/*is_object*/true))) { return false; } - if (JSON_HEDLEY_UNLIKELY(!check_bson_document_size(document_start, document_size))) - { - return false; - } + // the key currently being read; hoisted out of the loop so that its + // capacity is reused across elements and across nesting levels + string_t key; - return sax->end_object(); + while (true) + { + const auto element_type = get(); + + if (element_type == 0) // end of the innermost document + { + // a copy, not a reference: it must stay valid across the + // pop_back() below, which destroys the container_stack + // element it would otherwise alias + const container_frame top = container_stack.back(); + + if (JSON_HEDLEY_UNLIKELY(!check_bson_document_size(top.start_position, top.declared_size))) + { + return false; + } + + container_stack.pop_back(); + if (JSON_HEDLEY_UNLIKELY(top.is_object ? !sax->end_object() : !sax->end_array())) + { + return false; + } + // the document begun here is complete once it is not inside one + if (container_stack.empty()) + { + return true; + } + continue; + } + + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bson, "element list"))) + { + return false; + } + + const std::size_t element_type_parse_position = chars_read; + key.clear(); + if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key))) + { + return false; + } + + // an array's elements are named "0", "1", ... in the wire format, + // and those names are not passed on + if (container_stack.back().is_object && !sax->key(key)) + { + return false; + } + + if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_internal(element_type, element_type_parse_position))) + { + return false; + } + } } /*! @@ -419,12 +495,12 @@ class binary_reader case 0x03: // object { - return parse_bson_internal(); + return open_bson_document(/*is_object*/true); } case 0x04: // array { - return parse_bson_array(); + return open_bson_document(/*is_object*/false); } case 0x05: // binary @@ -474,82 +550,7 @@ class binary_reader } } - /*! - @brief Read a BSON element list (as specified in the BSON-spec) - The same binary layout is used for objects and arrays, hence it must be - indicated with the argument @a is_array which one is expected - (true --> array, false --> object). - - @param[in] is_array Determines if the element list being read is to be - treated as an object (@a is_array == false), or as an - array (@a is_array == true). - @return whether a valid BSON-object/array was passed to the SAX parser - */ - bool parse_bson_element_list(const bool is_array) - { - string_t key; - - while (auto element_type = get()) - { - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bson, "element list"))) - { - return false; - } - - const std::size_t element_type_parse_position = chars_read; - if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key))) - { - return false; - } - - if (!is_array && !sax->key(key)) - { - return false; - } - - if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_internal(element_type, element_type_parse_position))) - { - return false; - } - - // get_bson_cstr only appends - key.clear(); - } - - return true; - } - - /*! - @brief Reads an array from the BSON input and passes it to the SAX-parser. - @return whether a valid BSON-array was passed to the SAX parser - */ - bool parse_bson_array() - { - const std::size_t document_start = chars_read; - std::int32_t document_size{}; - if (!get_number(input_format_t::bson, document_size)) - { - return false; - } - - if (JSON_HEDLEY_UNLIKELY(!sax->start_array(detail::unknown_size()))) - { - return false; - } - - if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_list(/*is_array*/true))) - { - return false; - } - - if (JSON_HEDLEY_UNLIKELY(!check_bson_document_size(document_start, document_size))) - { - return false; - } - - return sax->end_array(); - } ////////// // CBOR // diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 295a0d47f..150118a1c 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -12155,9 +12155,14 @@ class binary_reader /// number of elements that have not been read yet, or npos when the /// container is not sized and ends at a marker instead std::size_t remaining; + /// BSON: value of chars_read before this document's size prefix, which + /// check_bson_document_size() needs once the document has been read + std::size_t start_position = 0; /// UBJSON/BJData: the type marker of an optimized container, so that /// its elements are read without one of their own; 0 otherwise char_int_type type_marker; + /// BSON: the size this document declares, in bytes + std::int32_t declared_size = 0; /// whether to close this container with end_object() or end_array() bool is_object; }; @@ -12232,8 +12237,10 @@ class binary_reader @brief Reads in a BSON-object and passes it to the SAX-parser. @return whether a valid BSON-value was passed to the SAX parser */ - bool parse_bson_internal() + bool open_bson_document(const bool is_object) { + // recorded before the size prefix is read, because + // check_bson_document_size() measures the document from here const std::size_t document_start = chars_read; std::int32_t document_size{}; if (!get_number(input_format_t::bson, document_size)) @@ -12241,22 +12248,91 @@ class binary_reader return false; } - if (JSON_HEDLEY_UNLIKELY(!sax->start_object(detail::unknown_size()))) + if (JSON_HEDLEY_UNLIKELY(!enter_container(is_object, detail::unknown_size()))) { return false; } - if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_list(/*is_array*/false))) + container_frame& frame = container_stack.back(); + frame.start_position = document_start; + frame.declared_size = document_size; + return true; + } + + /*! + @brief read a BSON document and everything nested inside it + + Reads elements until the document that was begun here is complete, + resuming the enclosing document each time an embedded one ends, so that + the nesting depth of the input costs heap rather than native stack + (see #5104). + + @return whether reading the document succeeded + */ + bool parse_bson_internal() + { + if (JSON_HEDLEY_UNLIKELY(!open_bson_document(/*is_object*/true))) { return false; } - if (JSON_HEDLEY_UNLIKELY(!check_bson_document_size(document_start, document_size))) - { - return false; - } + // the key currently being read; hoisted out of the loop so that its + // capacity is reused across elements and across nesting levels + string_t key; - return sax->end_object(); + while (true) + { + const auto element_type = get(); + + if (element_type == 0) // end of the innermost document + { + // a copy, not a reference: it must stay valid across the + // pop_back() below, which destroys the container_stack + // element it would otherwise alias + const container_frame top = container_stack.back(); + + if (JSON_HEDLEY_UNLIKELY(!check_bson_document_size(top.start_position, top.declared_size))) + { + return false; + } + + container_stack.pop_back(); + if (JSON_HEDLEY_UNLIKELY(top.is_object ? !sax->end_object() : !sax->end_array())) + { + return false; + } + // the document begun here is complete once it is not inside one + if (container_stack.empty()) + { + return true; + } + continue; + } + + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bson, "element list"))) + { + return false; + } + + const std::size_t element_type_parse_position = chars_read; + key.clear(); + if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key))) + { + return false; + } + + // an array's elements are named "0", "1", ... in the wire format, + // and those names are not passed on + if (container_stack.back().is_object && !sax->key(key)) + { + return false; + } + + if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_internal(element_type, element_type_parse_position))) + { + return false; + } + } } /*! @@ -12368,12 +12444,12 @@ class binary_reader case 0x03: // object { - return parse_bson_internal(); + return open_bson_document(/*is_object*/true); } case 0x04: // array { - return parse_bson_array(); + return open_bson_document(/*is_object*/false); } case 0x05: // binary @@ -12423,82 +12499,7 @@ class binary_reader } } - /*! - @brief Read a BSON element list (as specified in the BSON-spec) - The same binary layout is used for objects and arrays, hence it must be - indicated with the argument @a is_array which one is expected - (true --> array, false --> object). - - @param[in] is_array Determines if the element list being read is to be - treated as an object (@a is_array == false), or as an - array (@a is_array == true). - @return whether a valid BSON-object/array was passed to the SAX parser - */ - bool parse_bson_element_list(const bool is_array) - { - string_t key; - - while (auto element_type = get()) - { - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bson, "element list"))) - { - return false; - } - - const std::size_t element_type_parse_position = chars_read; - if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key))) - { - return false; - } - - if (!is_array && !sax->key(key)) - { - return false; - } - - if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_internal(element_type, element_type_parse_position))) - { - return false; - } - - // get_bson_cstr only appends - key.clear(); - } - - return true; - } - - /*! - @brief Reads an array from the BSON input and passes it to the SAX-parser. - @return whether a valid BSON-array was passed to the SAX parser - */ - bool parse_bson_array() - { - const std::size_t document_start = chars_read; - std::int32_t document_size{}; - if (!get_number(input_format_t::bson, document_size)) - { - return false; - } - - if (JSON_HEDLEY_UNLIKELY(!sax->start_array(detail::unknown_size()))) - { - return false; - } - - if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_list(/*is_array*/true))) - { - return false; - } - - if (JSON_HEDLEY_UNLIKELY(!check_bson_document_size(document_start, document_size))) - { - return false; - } - - return sax->end_array(); - } ////////// // CBOR // diff --git a/tests/src/unit-bson.cpp b/tests/src/unit-bson.cpp index f0b37ea3f..153e12d30 100644 --- a/tests/src/unit-bson.cpp +++ b/tests/src/unit-bson.cpp @@ -1150,6 +1150,91 @@ TEST_CASE("BSON document size mismatch") } } +TEST_CASE("BSON nesting does not consume the call stack") +{ + // An embedded document or array used to be read by calling back into the + // document reader, so the native call stack grew with the nesting depth of + // the input (#5104). The open documents are kept on a heap stack now. + // + // Deeply nested values must not be compared, copied or dumped here: those + // operations are still recursive and would reintroduce the crash. + + // A document nested deeply enough to have crashed. The bytes are built + // here rather than with to_bson(), because the writer still recurses once + // per level and would overflow the stack before the reader is ever + // reached. Every level is + // 0x03 'a' 0x00 0x00 + // so a level is eight bytes larger than the one it holds, and the sizes + // can be filled in from the outside in. + const std::size_t depth = 30000; + std::vector input; + input.reserve(5 + (8 * depth)); + for (std::size_t i = 0; i < depth; ++i) + { + const auto size = static_cast(5 + (8 * (depth - i))); + input.push_back(static_cast(size & 0xFF)); + input.push_back(static_cast((size >> 8) & 0xFF)); + input.push_back(static_cast((size >> 16) & 0xFF)); + input.push_back(static_cast((size >> 24) & 0xFF)); + input.push_back(0x03); // embedded document + input.push_back('a'); + input.push_back(0x00); + } + // the innermost document is empty, then one terminator closes each level + input.insert(input.end(), {0x05, 0x00, 0x00, 0x00, 0x00}); + input.insert(input.end(), depth, 0x00); + + SECTION("a well-formed deep document is read through the SAX interface") + { + SaxCountdown accept_all(1000000); + CHECK(json::sax_parse(input, &accept_all, json::input_format_t::bson)); + } + + SECTION("a well-formed deep document is read into a value") + { + json j = json::from_bson(input); + + // walked rather than compared: comparing, copying or dumping a value + // this deep is still recursive + std::size_t measured = 0; + const json* q = &j; + while (q->is_object() && !q->empty()) + { + q = &q->begin().value(); + ++measured; + } + CHECK(measured == depth); + } + + SECTION("embedded documents and arrays are still read the same way") + { + const json values = {{"a", {{"b", {{"c", 1}}}}}}; + CHECK(json::from_bson(json::to_bson(values)) == values); + + const json array = {{"a", {1, 2, 3}}}; + CHECK(json::from_bson(json::to_bson(array)) == array); + + const json mixed = {{"a", {json{{"x", 1}}, json{{"y", 2}}}}}; + CHECK(json::from_bson(json::to_bson(mixed)) == mixed); + + CHECK(json::from_bson(json::to_bson(json::object())) == json::object()); + } + + SECTION("a size that does not match is still reported per document") + { + // the embedded document claims one byte too many + std::vector const bad = + { + 0x15, 0x00, 0x00, 0x00, 0x03, 'a', 0x00, + 0x0D, 0x00, 0x00, 0x00, 0x08, 'b', 0x00, 0x01, 0x00, + 0x00 + }; + json _; + CHECK_THROWS_AS(_ = json::from_bson(bad), json::parse_error&); + CHECK(json::from_bson(bad, true, false).is_discarded()); + } +} + TEST_CASE("BSON numerical data") { SECTION("number")