From fb3dd97306b54c4e6d2c998e76dff1f5e9cb0bd9 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Mon, 20 Jul 2026 09:51:17 +0000 Subject: [PATCH] Route contiguous byte containers through the pointer adapter (fast paths in C++11) json::parse(std::string) - the most common entry point - did not benefit from the contiguous fast paths (bulk string scanning, UTF-8 bulk validation, memcpy for binary formats) in C++11..17: std::string::iterator is a library wrapper, not a raw pointer, and pre-C++20 there is no portable way to prove it contiguous, so supports_bulk_scan was false. Only raw pointers, string literals, and C-arrays (and, in C++20, anything modelling std::contiguous_iterator) took the fast path. Detect contiguous single-byte containers (std::string, std::vector, std::vector, std::string_view, ...) via is_contiguous_byte_ container and route them through an iterator_input_adapter built from data()/data()+size(). The generic iterator-based container overload is constrained to exclude these, so the two overloads are disjoint and there is no ambiguity (a plain competing overload loses to the greedy forwarding-reference container overload on reference binding, and a factory partial-specialization is ambiguous - both were tried and rejected). The pointer keeps the container's own element type, so char_type - and therefore all parsing behavior - is byte-for-byte identical to the iterator path (const char* for std::string, const std::uint8_t* for std::vector); only the raw pointer additionally turns on the fast paths. Lifetimes are unchanged: the container outlives the adapter for the full parse expression, exactly as the iterators it replaces did. Measured, C++11, json::parse/accept(std::string), g++ 13: long ASCII strings: accept 201 -> 3200 MB/s (~16x), parse 174 -> 1444 dense CJK: accept 263 -> 697 MB/s (~2.6x) short strings: accept 163 -> 243 MB/s (~1.5x) Verified: char_type preserved for std::string (char) and std::vector (uint8_t); CBOR/MsgPack round-trips from std::vector unchanged; 1,000,000 randomized documents accept and parse identically via std::string and via std::istream; deserialization/user-defined-input/parser/lexer/conversions/diagnostic- position suites pass (20,480 assertions); warning-clean on g++ and clang in C++11/17/20. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01AXcDtEma2PjxgmPS9cQGzA Signed-off-by: Niels Lohmann --- .../nlohmann/detail/input/input_adapters.hpp | 42 ++++++++++++++++++- single_include/nlohmann/json.hpp | 42 ++++++++++++++++++- 2 files changed, 80 insertions(+), 4 deletions(-) diff --git a/include/nlohmann/detail/input/input_adapters.hpp b/include/nlohmann/detail/input/input_adapters.hpp index 9d35f5547..7f87fc4c0 100644 --- a/include/nlohmann/detail/input/input_adapters.hpp +++ b/include/nlohmann/detail/input/input_adapters.hpp @@ -593,6 +593,24 @@ typename iterator_input_adapter_factory::adapter_typ return factory_type::create(first, last); } +// Detect a container that stores its elements contiguously as single bytes +// (std::string, std::vector, std::array, +// std::string_view, ...). Such inputs are wrapped in a pointer-based adapter so +// they benefit from the contiguous fast paths (bulk string scanning, memcpy for +// binary formats) in every C++ standard - not only in C++20, where the standard +// library iterators model std::contiguous_iterator and are detected directly. +template +struct is_contiguous_byte_container : std::false_type {}; + +template +struct is_contiguous_byte_container < ContainerType, void_t < +decltype(std::declval().data()), +decltype(std::declval().size()) >> + : std::integral_constant < bool, + std::is_pointer().data())>::value&& + std::is_integral().data())>::type>::value&& + sizeof(typename std::remove_pointer().data())>::type) == 1 > {}; + // Convenience shorthand from container to iterator // Enables ADL on begin(container) and end(container) // Encloses the using declarations in namespace for not to leak them to outside scope @@ -620,12 +638,32 @@ struct container_input_adapter_factory< ContainerType, } // namespace container_input_adapter_factory_impl -template -typename container_input_adapter_factory_impl::container_input_adapter_factory::adapter_type input_adapter(ContainerType&& container) +// General container path (iterator-based). Contiguous single-byte containers +// are excluded here and routed through the pointer-based overload below. +template < typename ContainerType, + enable_if_t < !is_contiguous_byte_container::value, int > = 0 > +typename container_input_adapter_factory_impl::container_input_adapter_factory::adapter_type input_adapter(ContainerType && container) { return container_input_adapter_factory_impl::container_input_adapter_factory::create(std::forward(container)); } +// Contiguous single-byte containers (std::string, std::vector, ...) are +// wrapped in a pointer-based adapter so the contiguous fast paths apply in every +// standard. The pointer keeps the container's own element type (const char* for +// std::string, const std::uint8_t* for std::vector, ...), so the +// resulting char_type - and therefore the parsing behavior - is byte-for-byte +// identical to the iterator-based path; only the raw pointer additionally +// enables the bulk fast paths. The container outlives the adapter for the whole +// parse (temporaries live until the end of the full expression), exactly as the +// iterators it replaces did. +template < typename ContainerType, + enable_if_t < is_contiguous_byte_container::value, int > = 0 > +auto input_adapter(ContainerType && container) +-> decltype(input_adapter(container.data(), container.data() + container.size())) +{ + return input_adapter(container.data(), container.data() + container.size()); +} + // specialization for std::string using string_input_adapter_type = decltype(input_adapter(std::declval())); diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 8e1b7a8ba..d3241cf62 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -7580,6 +7580,24 @@ typename iterator_input_adapter_factory::adapter_typ return factory_type::create(first, last); } +// Detect a container that stores its elements contiguously as single bytes +// (std::string, std::vector, std::array, +// std::string_view, ...). Such inputs are wrapped in a pointer-based adapter so +// they benefit from the contiguous fast paths (bulk string scanning, memcpy for +// binary formats) in every C++ standard - not only in C++20, where the standard +// library iterators model std::contiguous_iterator and are detected directly. +template +struct is_contiguous_byte_container : std::false_type {}; + +template +struct is_contiguous_byte_container < ContainerType, void_t < +decltype(std::declval().data()), +decltype(std::declval().size()) >> + : std::integral_constant < bool, + std::is_pointer().data())>::value&& + std::is_integral().data())>::type>::value&& + sizeof(typename std::remove_pointer().data())>::type) == 1 > {}; + // Convenience shorthand from container to iterator // Enables ADL on begin(container) and end(container) // Encloses the using declarations in namespace for not to leak them to outside scope @@ -7607,12 +7625,32 @@ struct container_input_adapter_factory< ContainerType, } // namespace container_input_adapter_factory_impl -template -typename container_input_adapter_factory_impl::container_input_adapter_factory::adapter_type input_adapter(ContainerType&& container) +// General container path (iterator-based). Contiguous single-byte containers +// are excluded here and routed through the pointer-based overload below. +template < typename ContainerType, + enable_if_t < !is_contiguous_byte_container::value, int > = 0 > +typename container_input_adapter_factory_impl::container_input_adapter_factory::adapter_type input_adapter(ContainerType && container) { return container_input_adapter_factory_impl::container_input_adapter_factory::create(std::forward(container)); } +// Contiguous single-byte containers (std::string, std::vector, ...) are +// wrapped in a pointer-based adapter so the contiguous fast paths apply in every +// standard. The pointer keeps the container's own element type (const char* for +// std::string, const std::uint8_t* for std::vector, ...), so the +// resulting char_type - and therefore the parsing behavior - is byte-for-byte +// identical to the iterator-based path; only the raw pointer additionally +// enables the bulk fast paths. The container outlives the adapter for the whole +// parse (temporaries live until the end of the full expression), exactly as the +// iterators it replaces did. +template < typename ContainerType, + enable_if_t < is_contiguous_byte_container::value, int > = 0 > +auto input_adapter(ContainerType && container) +-> decltype(input_adapter(container.data(), container.data() + container.size())) +{ + return input_adapter(container.data(), container.data() + container.size()); +} + // specialization for std::string using string_input_adapter_type = decltype(input_adapter(std::declval()));