From 7506a1b0bfb5f434fe0012b395e4789349e98112 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Fri, 23 Oct 2026 10:33:52 +0200 Subject: [PATCH] 2.5.7: share parse_array_index between json_pointer and json_view (#5891) Move json_pointer::parse_array_index() and its status enum to namespace detail (json_pointer keeps a forwarding member), and let the json_view pointer resolution use it instead of its own copy of the RFC 6901 index grammar. Both accepted the same tokens with the same outcome. Also update the comment on detail::decode(), which listed four of the five UTF-8 validators and now points to the test that keeps them in sync. Signed-off-by: Niels Lohmann --- include/nlohmann/detail/json_pointer.hpp | 181 ++++++++++++----------- include/nlohmann/detail/string_utils.hpp | 13 +- include/nlohmann/detail/view/pointer.hpp | 48 +----- 3 files changed, 112 insertions(+), 130 deletions(-) diff --git a/include/nlohmann/detail/json_pointer.hpp b/include/nlohmann/detail/json_pointer.hpp index a016ec00d..0c930c75a 100644 --- a/include/nlohmann/detail/json_pointer.hpp +++ b/include/nlohmann/detail/json_pointer.hpp @@ -38,6 +38,100 @@ NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail { struct json_pointer_access; + +/*! +@brief result of parse_array_index() + +json_pointer::array_index() maps each value to the corresponding +parse_error/out_of_range exception; json_pointer::contains() and +get_checked_or_null(), which must not throw for an out-of-range or +unrepresentable index, and the json_view pointer resolution switch on it +directly instead. +*/ +enum class array_index_status +{ + ok, ///< @a s is a valid, representable array index + leading_zero, ///< @a s begins with '0' but has more than one character + not_a_number, ///< @a s is neither empty nor "-" and does not begin with a digit + unresolved, ///< @a s could not be converted to an integer + exceeds_size_type ///< @a s converts to an integer that exceeds SizeType +}; + +/*! +@brief convert a JSON Pointer reference token into an array index (RFC 6901, Sect. 4) + +@param[in] s reference token to be converted into an array index; its data() + must be NUL-terminated +@param[out] idx the integer representation of @a s if array_index_status::ok + is returned; left unchanged otherwise + +@return whether @a s is a valid array index, and if not, why + +@note this function never throws; json_pointer::array_index(), the callers that + must not throw, and the json_view pointer resolution build on it instead + of each re-implementing the RFC 6901 digit rules and the range check +*/ +template +array_index_status parse_array_index(const StringType& s, SizeType& idx) noexcept +{ + // error condition (cf. RFC 6901, Sect. 4) + if (JSON_HEDLEY_UNLIKELY(s.size() > 1 && s[0] == '0')) + { + return array_index_status::leading_zero; + } + + // error condition (cf. RFC 6901, Sect. 4); this also covers single- + // character tokens, so "/x" and "/xy" fail alike; "-" and the empty + // token are left to the conversion below and are reported as unresolved + if (JSON_HEDLEY_UNLIKELY(!s.empty() && !(s.size() == 1 && s[0] == '-') && !(s[0] >= '0' && s[0] <= '9'))) + { + return array_index_status::not_a_number; + } + + // fast path: up to 18 decimal digits cannot overflow unsigned long long or + // SizeType (9 digits for a 32-bit SizeType), so strtoull and errno are not needed + constexpr std::size_t max_fast_digits = sizeof(SizeType) >= 8 ? 18 : 9; + if (!s.empty() && s.size() <= max_fast_digits) + { + SizeType acc = 0; + for (std::size_t i = 0; i < s.size(); ++i) + { + const char c = s[i]; + if (JSON_HEDLEY_UNLIKELY(c < '0' || c > '9')) + { + break; // not a plain number: let strtoull decide + } + acc = static_cast((acc * 10) + static_cast(c - '0')); + if (JSON_HEDLEY_UNLIKELY(i == s.size() - 1)) + { + idx = acc; + return array_index_status::ok; + } + } + } + + const char* p = s.data(); + char* p_end = nullptr; // NOLINT(misc-const-correctness) + errno = 0; // strtoull doesn't reset errno + const unsigned long long res = std::strtoull(p, &p_end, 10); // NOLINT(runtime/int) + if (p == p_end // invalid input or empty string + || errno == ERANGE // out of range + || JSON_HEDLEY_UNLIKELY(static_cast(p_end - p) != s.size())) // incomplete read + { + return array_index_status::unresolved; + } + + // the index does not fit into SizeType; on 64-bit platforms this is + // only SIZE_MAX itself (see #2203 and #5395) + if (res >= static_cast((std::numeric_limits::max)())) // NOLINT(runtime/int) + { + return array_index_status::exceeds_size_type; + } + + idx = static_cast(res); + return array_index_status::ok; +} + } // namespace detail /// @brief JSON Pointer defines a string syntax for identifying a specific value within a JSON document @@ -333,21 +427,7 @@ class json_pointer JSON_THROW(detail::out_of_range::create(detail::exception_id::pointer_unresolved, detail::concat("unresolved reference token '", reference_token, "'"), context)); } - /*! - @brief result of @ref parse_array_index - - @ref array_index maps each value to the corresponding parse_error/out_of_range - exception; @ref contains and @ref get_checked_or_null, which must not throw for - an out-of-range or unrepresentable index, switch on it directly instead. - */ - enum class array_index_status - { - ok, ///< @a s is a valid, representable array index - leading_zero, ///< @a s begins with '0' but has more than one character - not_a_number, ///< @a s is neither empty nor "-" and does not begin with a digit - unresolved, ///< @a s could not be converted to an integer - exceeds_size_type ///< @a s converts to an integer that exceeds size_type - }; + using array_index_status = detail::array_index_status; /// whether a reference token is "-" (a plain comparison with a string literal is not always inlined) static bool is_dash(const string_t& s) noexcept @@ -355,78 +435,11 @@ class json_pointer return s.size() == 1 && s[0] == '-'; } - /*! - @param[in] s reference token to be converted into an array index - @param[out] idx the integer representation of @a s if @ref array_index_status::ok - is returned; left unchanged otherwise - - @return whether @a s is a valid array index, and if not, why - - @note this function never throws; @ref array_index and the callers that must not - throw (@ref contains, @ref get_checked_or_null) build on it instead of each - re-implementing the RFC 6901 digit rules and the @a size_type range check - */ + /// detail::parse_array_index() for the size_type of @a BasicJsonType template static array_index_status parse_array_index(const string_t& s, typename BasicJsonType::size_type& idx) noexcept { - using size_type = typename BasicJsonType::size_type; - - // error condition (cf. RFC 6901, Sect. 4) - if (JSON_HEDLEY_UNLIKELY(s.size() > 1 && s[0] == '0')) - { - return array_index_status::leading_zero; - } - - // error condition (cf. RFC 6901, Sect. 4); this also covers single- - // character tokens, so "/x" and "/xy" fail alike; "-" and the empty - // token are left to the conversion below and are reported as unresolved - if (JSON_HEDLEY_UNLIKELY(!s.empty() && !is_dash(s) && !(s[0] >= '0' && s[0] <= '9'))) - { - return array_index_status::not_a_number; - } - - // fast path: up to 18 decimal digits cannot overflow unsigned long long or - // size_type (9 digits for a 32-bit size_type), so strtoull and errno are not needed - constexpr std::size_t max_fast_digits = sizeof(size_type) >= 8 ? 18 : 9; - if (!s.empty() && s.size() <= max_fast_digits) - { - size_type acc = 0; - for (std::size_t i = 0; i < s.size(); ++i) - { - const char c = s[i]; - if (JSON_HEDLEY_UNLIKELY(c < '0' || c > '9')) - { - break; // not a plain number: let strtoull decide - } - acc = static_cast((acc * 10) + static_cast(c - '0')); - if (JSON_HEDLEY_UNLIKELY(i == s.size() - 1)) - { - idx = acc; - return array_index_status::ok; - } - } - } - - const char* p = s.data(); - char* p_end = nullptr; // NOLINT(misc-const-correctness) - errno = 0; // strtoull doesn't reset errno - const unsigned long long res = std::strtoull(p, &p_end, 10); // NOLINT(runtime/int) - if (p == p_end // invalid input or empty string - || errno == ERANGE // out of range - || JSON_HEDLEY_UNLIKELY(static_cast(p_end - p) != s.size())) // incomplete read - { - return array_index_status::unresolved; - } - - // the index does not fit into size_type; on 64-bit platforms this is - // only SIZE_MAX itself (see #2203 and #5395) - if (res >= static_cast((std::numeric_limits::max)())) // NOLINT(runtime/int) - { - return array_index_status::exceeds_size_type; - } - - idx = static_cast(res); - return array_index_status::ok; + return detail::parse_array_index(s, idx); } /*! diff --git a/include/nlohmann/detail/string_utils.hpp b/include/nlohmann/detail/string_utils.hpp index 0fd041893..227c08db4 100644 --- a/include/nlohmann/detail/string_utils.hpp +++ b/include/nlohmann/detail/string_utils.hpp @@ -196,7 +196,7 @@ This is a single-byte step of a "shift-based" UTF-8 decoder originally written by Björn Hoehrmann. See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details. -The library checks UTF-8 well-formedness (RFC 3629, section 4) in three +The library checks UTF-8 well-formedness (RFC 3629, section 4) in five places, which differ in speed, diagnostics, and how they read the input: - decode() below: the serializer, to escape and, in strict mode, reject @@ -210,10 +210,15 @@ places, which differ in speed, diagnostics, and how they read the input: bulk string scan, the bulk path of the BON8 reader, and the BON8 writer. They must accept exactly what the lexer's switch accepts. - the byte path of binary_reader::get_bon8_string(): BON8 input without bulk - access, and the bytes the bulk path leaves to it. + access, and the bytes the bulk path leaves to it. It takes the valid range + of the second byte from utf8_shape(). +- check_utf8() in detail/view/edit.hpp: strings stored in an editable + json_document, with the index of the offending byte. It also uses + utf8_shape(). -All four must accept the same set of sequences, so a change to one needs a -matching change to the others. +All five must accept the same set of sequences, which the test "UTF-8 +validators accept the same sequences" in unit-class_lexer.cpp checks +exhaustively against decode() and utf8_shape(). @param[in,out] state the current decoder state @param[in,out] codep codepoint (valid only if resulting state is UTF8_ACCEPT) diff --git a/include/nlohmann/detail/view/pointer.hpp b/include/nlohmann/detail/view/pointer.hpp index 58a8ce043..be5aa14de 100644 --- a/include/nlohmann/detail/view/pointer.hpp +++ b/include/nlohmann/detail/view/pointer.hpp @@ -32,51 +32,15 @@ enum class pointer_mode contains, ///< as basic_json::contains(): no exceptions at all }; -/// the outcome of reading an array index from a reference token -enum class index_status -{ - ok, - leading_zero, ///< parse_error.106 - not_number, ///< parse_error.109 - unresolved, ///< out_of_range.404 - too_large, ///< out_of_range.410 -}; +/// the outcome of reading an array index: detail::parse_array_index(), shared with json_pointer +using index_status = detail::array_index_status; /// reads an array index like json_pointer::array_index (RFC 6901, Sect. 4), /// but reports errors instead of throwing them template index_status array_index(const StringType& s, std::size_t& idx) noexcept { - // as basic_json: the empty token and "-" (past the last element) are - // unresolved rather than not a number - if (s.empty() || (s.size() == 1 && s[0] == '-')) - { - return index_status::unresolved; - } - if (s[0] < '0' || s[0] > '9') - { - return index_status::not_number; - } - if (s.size() > 1 && s[0] == '0') - { - return index_status::leading_zero; - } - std::uint64_t v = 0; - for (const auto c : s) - { - const auto d = static_cast(static_cast(c)) - '0'; - if (d > 9 || v > ((std::numeric_limits::max)() - d) / 10) - { - return index_status::unresolved; // not a number, or beyond unsigned long long - } - v = (v * 10) + d; - } - if (v >= (std::numeric_limits::max)()) // (std::size_t converts to std::uint64_t implicitly) - { - return index_status::too_large; - } - idx = static_cast(v); - return index_status::ok; + return detail::parse_array_index(s, idx); } /// throws the exception json_pointer::array_index throws for this status @@ -87,9 +51,9 @@ template { case index_status::leading_zero: throw_parse_error(exception_id::pointer_index_leading_zero, concat("array index '", s, "' must not begin with '0'")); - case index_status::not_number: + case index_status::not_a_number: throw_parse_error(exception_id::pointer_index_not_a_number, concat("array index '", s, "' is not a number")); - case index_status::too_large: + case index_status::exceeds_size_type: throw_out_of_range(exception_id::value_out_of_range, concat("array index ", s, " exceeds size_type")); // LCOV_EXCL_LINE case index_status::unresolved: case index_status::ok: @@ -139,7 +103,7 @@ View resolve_pointer(View cur, const Tokens& tokens, pointer_mode mode) const index_status status = array_index(token, idx); if (status != index_status::ok) { - const bool parse_error = status == index_status::leading_zero || status == index_status::not_number; + const bool parse_error = status == index_status::leading_zero || status == index_status::not_a_number; if (throwing || (mode == pointer_mode::value && parse_error)) { throw_array_index_error(status, token);