Merge branch 'develop' into claude/noexception-value-json-pointer-5672

Conflicts:
- tests/src/unit-disabled_exceptions.cpp: kept both new sections (#5672 value(json_pointer) test and develop's ordered_json growth test)

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

docs/mkdocs/docs/api/basic_json/operator[].md

@@ -89,9 +89,6 @@ Strong exception safety: if an exception occurs, the original value stays intact
     - Throws [`out_of_range.410`](../../home/exceptions.md#jsonexceptionout_of_range410) if an array index in the passed
       JSON pointer `ptr` exceeds the range of `size_type` (e.g., on 32-bit platforms).
 
-    For the **const** version, an object key or array index in `ptr` that does not exist is not reported by an
-    exception, but is undefined behavior (see the notes below). Use [`at`](at.md) for checked access.
-
 ## Complexity
 
 1. Constant if `idx` is in the range of the array. Otherwise, linear in `idx - size()`.
@@ -106,12 +103,9 @@ Strong exception safety: if an exception occurs, the original value stays intact
     The following cases apply to the **const** overloads; the non-const overloads instead insert the missing element
     (see the notes below).
 
-    1. If the element at index `idx` does not exist, the behavior is undefined and is **guarded by a
-       [runtime assertion](../../features/assertions.md)**!
+    1. If the element at index `idx` does not exist, the behavior is undefined.
     2. If the element with key `key` does not exist, the behavior is undefined and is **guarded by a
        [runtime assertion](../../features/assertions.md)**!
-    3. If the JSON pointer `ptr` refers to an object key or an array index that does not exist, the behavior is
-       undefined and is **guarded by a [runtime assertion](../../features/assertions.md)**!
 
 1. The non-const version may add values: If `idx` is beyond the range of the array (i.e., `idx >= size()`), then the
    array is silently filled up with `#!json null` values to make `idx` a valid reference to the last stored element. In
@@ -267,11 +261,9 @@ Strong exception safety: if an exception occurs, the original value stays intact
 ## Version history
 
 1. Added in version 1.0.0. Fixed in version 3.13.0 to throw `#!cpp std::length_error` instead of emptying the array and
-   accessing it out of bounds when `idx` equals the maximum value of `size_type`. A missing index in the const version
-   is guarded by a runtime assertion since version 3.13.0.
+   accessing it out of bounds when `idx` equals the maximum value of `size_type`.
 2. Added in version 1.0.0. Added overloads for `T* key` in version 1.1.0. Removed overloads for `T* key` (replaced by 3)
    in version 3.11.0.
 3. Added in version 3.11.0. Fixed in version 3.13.0 to consistently accept `std::string_view`-convertible keys, as
    already supported by [`at`](at.md), [`value`](value.md), [`find`](find.md), and other lookup functions.
-4. Added in version 2.0.0. A missing array index in the const version is guarded by a runtime assertion since
-   version 3.13.0.
+4. Added in version 2.0.0.

docs/mkdocs/docs/features/assertions.md

@@ -16,15 +16,14 @@ before including the `json.hpp` header.
 
 ## Function with runtime assertions
 
-### Unchecked access to a const value
+### Unchecked object access to a const value
 
-Function [`operator[]`](../api/basic_json/operator%5B%5D.md) implements unchecked access for arrays and objects. Whereas
-a missing element is added in the case of non-const values, accessing a const value with a missing object key or an
-invalid array index is undefined behavior (think of a dereferenced null pointer) and yields a runtime assertion. This
-also applies to a [JSON pointer](json_pointer.md) that refers to a missing key or an invalid index.
+Function [`operator[]`](../api/basic_json/operator%5B%5D.md) implements unchecked access for objects. Whereas a missing
+key is added in the case of non-const objects, accessing a const object with a missing key is undefined behavior (think
+of a dereferenced null pointer) and yields a runtime assertion.
 
-If you are not sure whether an element exists, use checked access with the [`at` function](../api/basic_json/at.md)
-or call the [`contains` function](../api/basic_json/contains.md) before.
+If you are not sure whether an element in an object exists, use checked access with the
+[`at` function](../api/basic_json/at.md) or call the [`contains` function](../api/basic_json/contains.md) before.
 
 See also the documentation on [element access](element_access/index.md).
 
@@ -47,30 +46,7 @@ See also the documentation on [element access](element_access/index.md).
     Output:
 
     ```
-    Assertion failed: (it != m_data.m_value.object->end()), function operator[], file json.hpp, line 28795.
-    ```
-
-??? example "Example 2: Invalid array index in a JSON pointer"
-
-    The following code will trigger an assertion at runtime:
-
-    ```cpp
-    #include <nlohmann/json.hpp>
-    
-    using json = nlohmann::json;
-    using namespace nlohmann::literals;
-    
-    int main()
-    {
-        const json j = {{"array", {1, 2, 3}}};
-        auto v = j["/array/5"_json_pointer];
-    }
-    ```
-
-    Output:
-
-    ```
-    Assertion failed: (idx < m_data.m_value.array->size()), function operator[], file json.hpp, line 28758.
+    Assertion failed: (m_value.object->find(key) != m_value.object->end()), function operator[], file json.hpp, line 2144.
     ```
 
 ### Constructing from an uninitialized iterator range

include/nlohmann/detail/json_pointer.hpp

@@ -536,10 +536,6 @@ class json_pointer
     @return const reference to the JSON value pointed to by the JSON
     pointer
 
-    @pre Every object key and array index the pointer refers to exists.
-         Like the const operator[] for keys and indices, a missing one is
-         undefined behavior, guarded by a runtime assertion.
-
     @throw parse_error.106   if an array index begins with '0'
     @throw parse_error.109   if an array index was not a number
     @throw out_of_range.402  if the array index '-' is used
@@ -554,8 +550,7 @@ class json_pointer
             {
                 case detail::value_t::object:
                 {
-                    // use unchecked object access; the const operator[]
-                    // asserts that the key exists
+                    // use unchecked object access
                     ptr = &ptr->operator[](reference_token);
                     break;
                 }
@@ -568,8 +563,7 @@ class json_pointer
                         JSON_THROW(detail::out_of_range::create(402, detail::concat("array index '-' (", std::to_string(ptr->m_data.m_value.array->size()), ") is out of range"), ptr));
                     }
 
-                    // use unchecked array access; the const operator[]
-                    // asserts that the index exists
+                    // use unchecked array access
                     ptr = &ptr->operator[](array_index<BasicJsonType>(reference_token));
                     break;
                 }
@@ -685,19 +679,29 @@ class json_pointer
                         return nullptr;
                     }
 
-                    // may throw parse_error.106/109 for a malformed index; an
-                    // index that is syntactically valid but cannot be
-                    // represented (out_of_range.404/410) is treated like an
-                    // out-of-range index below
-                    typename BasicJsonType::size_type idx{};
-                    JSON_TRY
+                    // tokens that array_index() rejects with parse_error.106/109
+                    // are passed on to it; all other tokens that it would reject
+                    // with out_of_range.404/410 are detected here, so that this
+                    // also works without exceptions
+                    if (JSON_HEDLEY_UNLIKELY(reference_token.size() > 1 && !(reference_token[0] >= '1' && reference_token[0] <= '9')))
                     {
-                        idx = array_index<BasicJsonType>(reference_token);
+                        static_cast<void>(array_index<BasicJsonType>(reference_token)); // throws parse_error.106/109
                     }
-                    JSON_INTERNAL_CATCH (detail::out_of_range&)
+                    if (JSON_HEDLEY_UNLIKELY(reference_token.empty() || !std::all_of(reference_token.begin(), reference_token.end(), [](const char c)
+                {
+                    return c >= '0' && c <= '9';
+                })))
                     {
                         return nullptr;
                     }
+                    errno = 0; // strtoull() does not reset errno on success
+                    char* p_end = nullptr; // NOLINT(misc-const-correctness)
+                    const unsigned long long magnitude = std::strtoull(reference_token.data(), &p_end, 10); // NOLINT(runtime/int)
+                    if (JSON_HEDLEY_UNLIKELY(errno == ERANGE || magnitude >= static_cast<unsigned long long>((std::numeric_limits<typename BasicJsonType::size_type>::max)()))) // NOLINT(runtime/int)
+                    {
+                        return nullptr;
+                    }
+                    const auto idx = static_cast<typename BasicJsonType::size_type>(magnitude);
 
                     if (JSON_HEDLEY_UNLIKELY(idx >= ptr->m_data.m_value.array->size()))
                     {

include/nlohmann/json.hpp

@@ -2874,7 +2874,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
         // const operator[] only works for arrays
         if (JSON_HEDLEY_LIKELY(is_array()))
         {
-            JSON_ASSERT(idx < m_data.m_value.array->size());
             return m_data.m_value.array->operator[](idx);
         }
 

single_include/nlohmann/json.hpp

@@ -20148,10 +20148,6 @@ class json_pointer
     @return const reference to the JSON value pointed to by the JSON
     pointer
 
-    @pre Every object key and array index the pointer refers to exists.
-         Like the const operator[] for keys and indices, a missing one is
-         undefined behavior, guarded by a runtime assertion.
-
     @throw parse_error.106   if an array index begins with '0'
     @throw parse_error.109   if an array index was not a number
     @throw out_of_range.402  if the array index '-' is used
@@ -20166,8 +20162,7 @@ class json_pointer
             {
                 case detail::value_t::object:
                 {
-                    // use unchecked object access; the const operator[]
-                    // asserts that the key exists
+                    // use unchecked object access
                     ptr = &ptr->operator[](reference_token);
                     break;
                 }
@@ -20180,8 +20175,7 @@ class json_pointer
                         JSON_THROW(detail::out_of_range::create(402, detail::concat("array index '-' (", std::to_string(ptr->m_data.m_value.array->size()), ") is out of range"), ptr));
                     }
 
-                    // use unchecked array access; the const operator[]
-                    // asserts that the index exists
+                    // use unchecked array access
                     ptr = &ptr->operator[](array_index<BasicJsonType>(reference_token));
                     break;
                 }
@@ -20297,19 +20291,29 @@ class json_pointer
                         return nullptr;
                     }
 
-                    // may throw parse_error.106/109 for a malformed index; an
-                    // index that is syntactically valid but cannot be
-                    // represented (out_of_range.404/410) is treated like an
-                    // out-of-range index below
-                    typename BasicJsonType::size_type idx{};
-                    JSON_TRY
+                    // tokens that array_index() rejects with parse_error.106/109
+                    // are passed on to it; all other tokens that it would reject
+                    // with out_of_range.404/410 are detected here, so that this
+                    // also works without exceptions
+                    if (JSON_HEDLEY_UNLIKELY(reference_token.size() > 1 && !(reference_token[0] >= '1' && reference_token[0] <= '9')))
                     {
-                        idx = array_index<BasicJsonType>(reference_token);
+                        static_cast<void>(array_index<BasicJsonType>(reference_token)); // throws parse_error.106/109
                     }
-                    JSON_INTERNAL_CATCH (detail::out_of_range&)
+                    if (JSON_HEDLEY_UNLIKELY(reference_token.empty() || !std::all_of(reference_token.begin(), reference_token.end(), [](const char c)
+                {
+                    return c >= '0' && c <= '9';
+                })))
                     {
                         return nullptr;
                     }
+                    errno = 0; // strtoull() does not reset errno on success
+                    char* p_end = nullptr; // NOLINT(misc-const-correctness)
+                    const unsigned long long magnitude = std::strtoull(reference_token.data(), &p_end, 10); // NOLINT(runtime/int)
+                    if (JSON_HEDLEY_UNLIKELY(errno == ERANGE || magnitude >= static_cast<unsigned long long>((std::numeric_limits<typename BasicJsonType::size_type>::max)()))) // NOLINT(runtime/int)
+                    {
+                        return nullptr;
+                    }
+                    const auto idx = static_cast<typename BasicJsonType::size_type>(magnitude);
 
                     if (JSON_HEDLEY_UNLIKELY(idx >= ptr->m_data.m_value.array->size()))
                     {
@@ -29807,7 +29811,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
         // const operator[] only works for arrays
         if (JSON_HEDLEY_LIKELY(is_array()))
         {
-            JSON_ASSERT(idx < m_data.m_value.array->size());
             return m_data.m_value.array->operator[](idx);
         }
 

tests/src/unit-disabled_exceptions.cpp

@@ -47,6 +47,21 @@ TEST_CASE("Tests with disabled exceptions")
         delete sax_no_exception::error_string; // NOLINT(cppcoreguidelines-owning-memory)
     }
 
+    SECTION("issue #5672 - value(json_pointer, default) must not abort for array tokens that are not a valid index")
+    {
+        const json j = {1, 2, 3};
+
+        // a syntactically valid index that is out of range for this array
+        CHECK(j.value("/7"_json_pointer, 42) == 42);
+        // a reference token that is not a number at all
+        CHECK(j.value("/1a"_json_pointer, 42) == 42);
+        // the empty reference token (JSON pointer "/")
+        CHECK(j.value("/"_json_pointer, 42) == 42);
+        // an index whose magnitude does not fit into size_type
+        CHECK(j.value("/99999999999999999999999"_json_pointer, 42) == 42);
+        CHECK(j.value("/18446744073709551615"_json_pointer, 42) == 42);
+    }
+
     SECTION("growing an ordered_json object")
     {
         auto j = nlohmann::ordered_json::object();

tests/src/unit-element_access2.cpp

@@ -516,6 +516,21 @@ TEST_CASE_TEMPLATE("element access 2", Json, nlohmann::json, nlohmann::ordered_j
                     CHECK(j_array.value("/-"_json_pointer, 42) == 42);
                     CHECK(j_array_const.value("/-"_json_pointer, 42) == 42);
 
+                    // Test an index with a non-digit after a valid leading digit; this is
+                    // out_of_range (not parse_error) and must not throw (see #5672)
+                    CHECK(j_array.value("/1a"_json_pointer, 42) == 42);
+                    CHECK(j_array_const.value("/1a"_json_pointer, 42) == 42);
+
+                    // Test the empty reference token (JSON pointer "/"); see #5672
+                    CHECK(j_array.value("/"_json_pointer, 42) == 42);
+                    CHECK(j_array_const.value("/"_json_pointer, 42) == 42);
+
+                    // Test an index whose magnitude does not fit into size_type (see #5672)
+                    CHECK(j_array.value("/99999999999999999999999"_json_pointer, 42) == 42);
+                    CHECK(j_array_const.value("/99999999999999999999999"_json_pointer, 42) == 42);
+                    CHECK(j_array.value("/18446744073709551615"_json_pointer, 42) == 42);
+                    CHECK(j_array_const.value("/18446744073709551615"_json_pointer, 42) == 42);
+
 #if !defined(JSON_NOEXCEPTION)
                     // Test malformed index (non-numeric) throws parse_error
                     CHECK_THROWS_WITH_AS(j_array.value("/foo"_json_pointer, 1), "[json.exception.parse_error.109] parse error: array index 'foo' is not a number", typename Json::parse_error&);