From 424a88b4fbca151202f3bbb444fbf9386c42c7d8 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sat, 11 Jul 2026 16:25:57 +0200 Subject: [PATCH] Extend memcpy fast path to sized sentinels (e.g. std::counted_iterator) Extend the memcpy fast path in iterator_input_adapter to sized sentinels of a different type, not just same-type iterator pairs. std::counted_iterator paired with std::default_sentinel_t already satisfies std::contiguous_iterator and std::sized_sentinel_for, so std::ranges::distance (C++20) lets that combination reach the fast path too, instead of silently falling back to the byte-by-byte path. - iterator_is_contiguous now also allows std::sized_sentinel_for under C++20, gated the same way as the existing std::contiguous_iterator detection. - get_elements_impl's fast path uses std::ranges::distance under C++20 (works for both same-type and sized-sentinel pairs) and falls back to std::distance pre-C++20, where SentinelType is always IteratorType. - Add a C++20-only test exercising json::parse/accept with std::counted_iterator + std::default_sentinel_t. - Document std::default_sentinel_t + std::counted_iterator as a SentinelType example across parse.md, accept.md, sax_parse.md, and the five from_*.md pages, replacing an earlier ambiguously worded bullet. Addresses review feedback: https://github.com/nlohmann/json/pull/5265#discussion_r3564237584 Signed-off-by: Niels Lohmann --- docs/mkdocs/docs/api/basic_json/accept.md | 2 +- .../mkdocs/docs/api/basic_json/from_bjdata.md | 6 ++-- docs/mkdocs/docs/api/basic_json/from_bson.md | 6 ++-- docs/mkdocs/docs/api/basic_json/from_cbor.md | 6 ++-- .../docs/api/basic_json/from_msgpack.md | 6 ++-- .../mkdocs/docs/api/basic_json/from_ubjson.md | 6 ++-- docs/mkdocs/docs/api/basic_json/parse.md | 2 +- docs/mkdocs/docs/api/basic_json/sax_parse.md | 5 +++- .../nlohmann/detail/input/input_adapters.hpp | 17 ++++++++--- single_include/nlohmann/json.hpp | 17 ++++++++--- tests/src/unit-user_defined_input.cpp | 28 +++++++++++++++++++ 11 files changed, 80 insertions(+), 21 deletions(-) diff --git a/docs/mkdocs/docs/api/basic_json/accept.md b/docs/mkdocs/docs/api/basic_json/accept.md index dac5f9f36..5ba009c3e 100644 --- a/docs/mkdocs/docs/api/basic_json/accept.md +++ b/docs/mkdocs/docs/api/basic_json/accept.md @@ -49,7 +49,7 @@ Unlike the [`parse()`](parse.md) function, this function neither throws an excep : defaults to `IteratorType`; may be a different type comparable to `IteratorType` via `operator!=`, for instance. - a custom sentinel type for C++20 ranges - - `std::counted_iterator` with a different sentinel type + - `std::default_sentinel_t`, when `IteratorType` is `std::counted_iterator` ## Parameters diff --git a/docs/mkdocs/docs/api/basic_json/from_bjdata.md b/docs/mkdocs/docs/api/basic_json/from_bjdata.md index 150f17aac..ac60bb58b 100644 --- a/docs/mkdocs/docs/api/basic_json/from_bjdata.md +++ b/docs/mkdocs/docs/api/basic_json/from_bjdata.md @@ -36,8 +36,10 @@ The exact mapping and its limitations are described on a [dedicated page](../../ : a compatible iterator type `SentinelType` -: defaults to `IteratorType`; may be a different type comparable to `IteratorType` via `operator!=`, for instance a - custom sentinel type for C++20 ranges +: defaults to `IteratorType`; may be a different type comparable to `IteratorType` via `operator!=`, for instance. + + - a custom sentinel type for C++20 ranges + - `std::default_sentinel_t`, when `IteratorType` is `std::counted_iterator` ## Parameters diff --git a/docs/mkdocs/docs/api/basic_json/from_bson.md b/docs/mkdocs/docs/api/basic_json/from_bson.md index eb06a7177..ce093a0d8 100644 --- a/docs/mkdocs/docs/api/basic_json/from_bson.md +++ b/docs/mkdocs/docs/api/basic_json/from_bson.md @@ -36,8 +36,10 @@ The exact mapping and its limitations are described on a [dedicated page](../../ : a compatible iterator type `SentinelType` -: defaults to `IteratorType`; may be a different type comparable to `IteratorType` via `operator!=`, for instance a - custom sentinel type for C++20 ranges +: defaults to `IteratorType`; may be a different type comparable to `IteratorType` via `operator!=`, for instance. + + - a custom sentinel type for C++20 ranges + - `std::default_sentinel_t`, when `IteratorType` is `std::counted_iterator` ## Parameters diff --git a/docs/mkdocs/docs/api/basic_json/from_cbor.md b/docs/mkdocs/docs/api/basic_json/from_cbor.md index 3792307bd..fb28edd3b 100644 --- a/docs/mkdocs/docs/api/basic_json/from_cbor.md +++ b/docs/mkdocs/docs/api/basic_json/from_cbor.md @@ -39,8 +39,10 @@ The exact mapping and its limitations are described on a [dedicated page](../../ : a compatible iterator type `SentinelType` -: defaults to `IteratorType`; may be a different type comparable to `IteratorType` via `operator!=`, for instance a - custom sentinel type for C++20 ranges +: defaults to `IteratorType`; may be a different type comparable to `IteratorType` via `operator!=`, for instance. + + - a custom sentinel type for C++20 ranges + - `std::default_sentinel_t`, when `IteratorType` is `std::counted_iterator` ## Parameters diff --git a/docs/mkdocs/docs/api/basic_json/from_msgpack.md b/docs/mkdocs/docs/api/basic_json/from_msgpack.md index dd7719804..12834e2c3 100644 --- a/docs/mkdocs/docs/api/basic_json/from_msgpack.md +++ b/docs/mkdocs/docs/api/basic_json/from_msgpack.md @@ -36,8 +36,10 @@ The exact mapping and its limitations are described on a [dedicated page](../../ : a compatible iterator type `SentinelType` -: defaults to `IteratorType`; may be a different type comparable to `IteratorType` via `operator!=`, for instance a - custom sentinel type for C++20 ranges +: defaults to `IteratorType`; may be a different type comparable to `IteratorType` via `operator!=`, for instance. + + - a custom sentinel type for C++20 ranges + - `std::default_sentinel_t`, when `IteratorType` is `std::counted_iterator` ## Parameters diff --git a/docs/mkdocs/docs/api/basic_json/from_ubjson.md b/docs/mkdocs/docs/api/basic_json/from_ubjson.md index 55c68c0ae..a0f8944aa 100644 --- a/docs/mkdocs/docs/api/basic_json/from_ubjson.md +++ b/docs/mkdocs/docs/api/basic_json/from_ubjson.md @@ -36,8 +36,10 @@ The exact mapping and its limitations are described on a [dedicated page](../../ : a compatible iterator type `SentinelType` -: defaults to `IteratorType`; may be a different type comparable to `IteratorType` via `operator!=`, for instance a - custom sentinel type for C++20 ranges +: defaults to `IteratorType`; may be a different type comparable to `IteratorType` via `operator!=`, for instance. + + - a custom sentinel type for C++20 ranges + - `std::default_sentinel_t`, when `IteratorType` is `std::counted_iterator` ## Parameters diff --git a/docs/mkdocs/docs/api/basic_json/parse.md b/docs/mkdocs/docs/api/basic_json/parse.md index 4565c8762..ef22e9873 100644 --- a/docs/mkdocs/docs/api/basic_json/parse.md +++ b/docs/mkdocs/docs/api/basic_json/parse.md @@ -48,7 +48,7 @@ static basic_json parse(IteratorType first, SentinelType last, : defaults to `IteratorType`; may be a different type comparable to `IteratorType` via `operator!=`, for instance. - a custom sentinel type for C++20 ranges - - `std::counted_iterator` with a different sentinel type + - `std::default_sentinel_t`, when `IteratorType` is `std::counted_iterator` ## Parameters diff --git a/docs/mkdocs/docs/api/basic_json/sax_parse.md b/docs/mkdocs/docs/api/basic_json/sax_parse.md index 606ab9bbe..fc8ce07b6 100644 --- a/docs/mkdocs/docs/api/basic_json/sax_parse.md +++ b/docs/mkdocs/docs/api/basic_json/sax_parse.md @@ -48,7 +48,10 @@ The SAX event lister must follow the interface of [`json_sax`](../json_sax/index with a size of 1, 2, or 4 bytes (interpreted respectively as UTF-8, UTF-16, and UTF-32) `SentinelType` -: defaults to `IteratorType`; may be a different type comparable to `IteratorType` via `operator!=`, for overload (2) +: defaults to `IteratorType`; may be a different type comparable to `IteratorType` via `operator!=`, for overload (2), for instance. + + - a custom sentinel type for C++20 ranges + - `std::default_sentinel_t`, when `IteratorType` is `std::counted_iterator` `SAX` : a class fulfilling the SAX event listener interface; see [`json_sax`](../json_sax/index.md) diff --git a/include/nlohmann/detail/input/input_adapters.hpp b/include/nlohmann/detail/input/input_adapters.hpp index 1ef8a6af7..ab9173613 100644 --- a/include/nlohmann/detail/input/input_adapters.hpp +++ b/include/nlohmann/detail/input/input_adapters.hpp @@ -220,20 +220,29 @@ class iterator_input_adapter // whether IteratorType refers to a contiguous range and therefore supports // a std::memcpy fast path (pointers always do; in C++20 we can also detect // library iterators such as those of std::vector and std::string). - // The fast path also requires SentinelType == IteratorType so std::distance works. + // Computing the available element count needs either same-type iterators + // (plain std::distance) or, in C++20, a sized sentinel (std::ranges::distance), + // e.g. std::counted_iterator paired with std::default_sentinel_t. static constexpr bool iterator_is_contiguous = - std::is_same::value && ( #if defined(__cpp_lib_concepts) && defined(JSON_HAS_CPP_20) - std::contiguous_iterator || + (std::is_same::value || std::sized_sentinel_for) + && (std::contiguous_iterator || std::is_pointer::value); +#else + std::is_same::value && std::is_pointer::value; #endif - std::is_pointer::value); // contiguous fast path: bulk copy the remaining range with std::memcpy template std::size_t get_elements_impl(T* dest, std::size_t count, std::true_type /*contiguous*/) { const std::size_t wanted = count * sizeof(T); +#if defined(__cpp_lib_concepts) && defined(JSON_HAS_CPP_20) + // std::ranges::distance also supports sized sentinels of a different + // type (e.g. std::counted_iterator + std::default_sentinel_t) + const std::size_t available = static_cast(std::ranges::distance(current, end)) * sizeof(char_type); +#else const std::size_t available = static_cast(std::distance(current, end)) * sizeof(char_type); +#endif const std::size_t copied = (std::min)(wanted, available); if (JSON_HEDLEY_LIKELY(copied != 0)) { diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 99fceb63b..03d235173 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -7207,20 +7207,29 @@ class iterator_input_adapter // whether IteratorType refers to a contiguous range and therefore supports // a std::memcpy fast path (pointers always do; in C++20 we can also detect // library iterators such as those of std::vector and std::string). - // The fast path also requires SentinelType == IteratorType so std::distance works. + // Computing the available element count needs either same-type iterators + // (plain std::distance) or, in C++20, a sized sentinel (std::ranges::distance), + // e.g. std::counted_iterator paired with std::default_sentinel_t. static constexpr bool iterator_is_contiguous = - std::is_same::value && ( #if defined(__cpp_lib_concepts) && defined(JSON_HAS_CPP_20) - std::contiguous_iterator || + (std::is_same::value || std::sized_sentinel_for) + && (std::contiguous_iterator || std::is_pointer::value); +#else + std::is_same::value && std::is_pointer::value; #endif - std::is_pointer::value); // contiguous fast path: bulk copy the remaining range with std::memcpy template std::size_t get_elements_impl(T* dest, std::size_t count, std::true_type /*contiguous*/) { const std::size_t wanted = count * sizeof(T); +#if defined(__cpp_lib_concepts) && defined(JSON_HAS_CPP_20) + // std::ranges::distance also supports sized sentinels of a different + // type (e.g. std::counted_iterator + std::default_sentinel_t) + const std::size_t available = static_cast(std::ranges::distance(current, end)) * sizeof(char_type); +#else const std::size_t available = static_cast(std::distance(current, end)) * sizeof(char_type); +#endif const std::size_t copied = (std::min)(wanted, available); if (JSON_HEDLEY_LIKELY(copied != 0)) { diff --git a/tests/src/unit-user_defined_input.cpp b/tests/src/unit-user_defined_input.cpp index cb93439e5..f9485fcf7 100644 --- a/tests/src/unit-user_defined_input.cpp +++ b/tests/src/unit-user_defined_input.cpp @@ -6,6 +6,13 @@ // SPDX-FileCopyrightText: 2013-2026 Niels Lohmann // SPDX-License-Identifier: MIT +// cmake/test.cmake selects the C++ standard versions with which to build a +// unit test based on the presence of JSON_HAS_CPP_ macros. +// When using macros that are only defined for particular versions of the standard +// (e.g., JSON_HAS_FILESYSTEM for C++17 and up), please mention the corresponding +// version macro in a comment close by, like this: +// JSON_HAS_CPP_ (do not remove; see note at top of file) + #include "doctest_compatibility.h" #include @@ -13,6 +20,10 @@ using nlohmann::json; #include +#if defined(__cpp_lib_concepts) && defined(JSON_HAS_CPP_20) + #include +#endif + namespace { TEST_CASE("Use arbitrary stdlib container") @@ -201,4 +212,21 @@ TEST_CASE("Parse with heterogeneous iterator and sentinel types") CHECK(j2.at(0) == 1); } +#if defined(__cpp_lib_concepts) && defined(JSON_HAS_CPP_20) +// JSON_HAS_CPP_20 (do not remove; see note at top of file) +TEST_CASE("Parse with std::counted_iterator and std::default_sentinel_t") +{ + const std::string json_str = R"({"key":"value","array":[1,2,3]})"; + const auto len = static_cast>(json_str.size()); + + std::counted_iterator first(json_str.begin(), len); + const json j = json::parse(first, std::default_sentinel); + CHECK(j["key"] == "value"); + CHECK(j["array"].size() == 3); + + std::counted_iterator first2(json_str.begin(), len); + CHECK(json::accept(first2, std::default_sentinel)); +} +#endif + } // namespace