json_view: no byte range in exceptions about a removed value

#5990 dropped the found flag from diagnostics(), reasoning that a value
value_path() does not find leaves no tokens, like the root. But the node
of a value that is no longer part of the document still holds the bytes
it had in the source, so with JSON_DIAGNOSTICS and
JSON_DIAGNOSTIC_POSITIONS its exceptions gained "(bytes 1-4)" for text
that is no longer there. The test #5997 added for exactly this case
(unit-json_view_positions.cpp, "views of values that are no longer part
of the document") has failed on develop since both were merged.

Restore the flag: a value that is not found gets no context, the root
keeps its byte range.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

include/nlohmann/detail/input/binary_reader.hpp

@@ -1024,7 +1024,7 @@ class binary_reader
             {
                 std::uint64_t len{};
                 std::size_t size{};
-                return get_cbor_argument(len) && get_container_size(len, size, "array") && enter_cbor_array(size);
+                return get_cbor_argument(len) && get_cbor_container_size(len, size, "array") && enter_cbor_array(size);
             }
 
             case 0x9F: // array (indefinite length)
@@ -1064,7 +1064,7 @@ class binary_reader
             {
                 std::uint64_t len{};
                 std::size_t size{};
-                return get_cbor_argument(len) && get_container_size(len, size, "map") && enter_object(size);
+                return get_cbor_argument(len) && get_cbor_container_size(len, size, "map") && enter_object(size);
             }
 
             case 0xBF: // map (indefinite length)
@@ -1700,7 +1700,7 @@ class binary_reader
     }
 
     /*!
-    @brief narrow a definite CBOR or MessagePack array/map length to std::size_t
+    @brief narrow a definite CBOR array/map length to std::size_t
 
     A definite length is rejected if it does not fit in std::size_t or if it
     equals detail::unknown_size(), which is reserved to mark an indefinite-
@@ -1713,7 +1713,7 @@ class binary_reader
     @param[in]  context  "array" or "map", for the error message
     @return whether the length is usable
     */
-    bool get_container_size(const std::uint64_t len, std::size_t& result, const char* context)
+    bool get_cbor_container_size(const std::uint64_t len, std::size_t& result, const char* context)
     {
         if (JSON_HEDLEY_UNLIKELY(!value_in_range_of<std::size_t>(len) || len == detail::unknown_size()))
         {
@@ -2309,8 +2309,7 @@ class binary_reader
             case 0xDD: // array 32
             {
                 std::uint32_t len{};
-                std::size_t size{};
-                return get_number(len) && get_container_size(len, size, "array") && enter_msgpack_array(size);
+                return get_number(len) && enter_msgpack_array(conditional_static_cast<std::size_t>(len));
             }
 
             case 0xDE: // map 16
@@ -2322,8 +2321,7 @@ class binary_reader
             case 0xDF: // map 32
             {
                 std::uint32_t len{};
-                std::size_t size{};
-                return get_number(len) && get_container_size(len, size, "map") && enter_object(size);
+                return get_number(len) && enter_object(conditional_static_cast<std::size_t>(len));
             }
 
             // negative fixint
@@ -3133,27 +3131,6 @@ class binary_reader
         return true;
     }
 
-    /*!
-    @brief store the count of an optimized container (or the length of a string)
-
-    A count equal to npos is rejected: npos marks a container whose count is
-    not known, so a count of that value would be read as "no count" (compare
-    get_container_size()). No container can hold that many elements.
-
-    @param[in]  count   the count read from the input
-    @param[out] result  set to @a count unless it is rejected
-    @return whether @a count is usable
-    */
-    bool store_ubjson_count(const std::size_t count, std::size_t& result)
-    {
-        if (JSON_HEDLEY_UNLIKELY(count == npos))
-        {
-            return out_of_range_error(exception_id::container_too_large, "integer value overflow", "size");
-        }
-        result = count;
-        return true;
-    }
-
     /*!
     @brief read a UBJSON/BJData optimized-container count of a signed marker
            type ('i', 'I', 'l', 'L') and narrow it to std::size_t
@@ -3184,7 +3161,8 @@ class binary_reader
         {
             return out_of_range_error(exception_id::container_too_large, "integer value overflow", "size");
         }
-        return store_ubjson_count(static_cast<std::size_t>(number), result); // NOLINT(bugprone-signed-char-misuse,cert-str34-c): number is not a char
+        result = static_cast<std::size_t>(number); // NOLINT(bugprone-signed-char-misuse,cert-str34-c): number is not a char
+        return true;
     }
 
     /*!
@@ -3261,7 +3239,8 @@ class binary_reader
                 {
                     return false;
                 }
-                return store_ubjson_count(conditional_static_cast<std::size_t>(number), result);
+                result = conditional_static_cast<std::size_t>(number);
+                return true;
             }
 
             case 'M':
@@ -3279,7 +3258,8 @@ class binary_reader
                 {
                     return out_of_range_error(exception_id::container_too_large, "integer value overflow", "size");
                 }
-                return store_ubjson_count(detail::conditional_static_cast<std::size_t>(number), result);
+                result = detail::conditional_static_cast<std::size_t>(number);
+                return true;
             }
 
             case '[':

include/nlohmann/detail/view/errors.hpp

@@ -230,7 +230,8 @@ void value_range(const document_data& d, const node* n, std::size_t& start, std:
 
 /// the context basic_json puts in front of the message of an exception about
 /// a value: "(/path) " with JSON_DIAGNOSTICS (none for the root), then
-/// "(bytes a-b) " with JSON_DIAGNOSTIC_POSITIONS (also for the root)
+/// "(bytes a-b) " with JSON_DIAGNOSTIC_POSITIONS (also for the root); with
+/// JSON_DIAGNOSTICS, nothing for a value that is no longer part of the document
 ///
 /// This and the throw helpers that take an error_site are templates, so that
 /// the path and range code is instantiated only where an exception can be
@@ -251,8 +252,14 @@ std::string diagnostics(const error_site site)
     std::string context;
 #if JSON_DIAGNOSTICS
     std::vector<std::string> tokens;
-    // a value that is not found leaves no tokens, like the root
-    static_cast<void>(edited ? value_path<true>(d, site.value(), tokens) : value_path<false>(d, site.value(), tokens));
+    // a value that is no longer part of the document has no path, and its
+    // node still holds the bytes it had in the source: no context at all
+    const bool found = edited ? value_path<true>(d, site.value(), tokens) : value_path<false>(d, site.value(), tokens);
+    if (!found)
+    {
+        // context is still empty; returning it keeps the copy elided (-Wnrvo)
+        return context;
+    }
     if (!tokens.empty())
     {
         // the root value has no path, but still has a byte range

single_include/nlohmann/json.hpp

@@ -19039,7 +19039,7 @@ class binary_reader
             {
                 std::uint64_t len{};
                 std::size_t size{};
-                return get_cbor_argument(len) && get_container_size(len, size, "array") && enter_cbor_array(size);
+                return get_cbor_argument(len) && get_cbor_container_size(len, size, "array") && enter_cbor_array(size);
             }
 
             case 0x9F: // array (indefinite length)
@@ -19079,7 +19079,7 @@ class binary_reader
             {
                 std::uint64_t len{};
                 std::size_t size{};
-                return get_cbor_argument(len) && get_container_size(len, size, "map") && enter_object(size);
+                return get_cbor_argument(len) && get_cbor_container_size(len, size, "map") && enter_object(size);
             }
 
             case 0xBF: // map (indefinite length)
@@ -19715,7 +19715,7 @@ class binary_reader
     }
 
     /*!
-    @brief narrow a definite CBOR or MessagePack array/map length to std::size_t
+    @brief narrow a definite CBOR array/map length to std::size_t
 
     A definite length is rejected if it does not fit in std::size_t or if it
     equals detail::unknown_size(), which is reserved to mark an indefinite-
@@ -19728,7 +19728,7 @@ class binary_reader
     @param[in]  context  "array" or "map", for the error message
     @return whether the length is usable
     */
-    bool get_container_size(const std::uint64_t len, std::size_t& result, const char* context)
+    bool get_cbor_container_size(const std::uint64_t len, std::size_t& result, const char* context)
     {
         if (JSON_HEDLEY_UNLIKELY(!value_in_range_of<std::size_t>(len) || len == detail::unknown_size()))
         {
@@ -20324,8 +20324,7 @@ class binary_reader
             case 0xDD: // array 32
             {
                 std::uint32_t len{};
-                std::size_t size{};
-                return get_number(len) && get_container_size(len, size, "array") && enter_msgpack_array(size);
+                return get_number(len) && enter_msgpack_array(conditional_static_cast<std::size_t>(len));
             }
 
             case 0xDE: // map 16
@@ -20337,8 +20336,7 @@ class binary_reader
             case 0xDF: // map 32
             {
                 std::uint32_t len{};
-                std::size_t size{};
-                return get_number(len) && get_container_size(len, size, "map") && enter_object(size);
+                return get_number(len) && enter_object(conditional_static_cast<std::size_t>(len));
             }
 
             // negative fixint
@@ -21148,27 +21146,6 @@ class binary_reader
         return true;
     }
 
-    /*!
-    @brief store the count of an optimized container (or the length of a string)
-
-    A count equal to npos is rejected: npos marks a container whose count is
-    not known, so a count of that value would be read as "no count" (compare
-    get_container_size()). No container can hold that many elements.
-
-    @param[in]  count   the count read from the input
-    @param[out] result  set to @a count unless it is rejected
-    @return whether @a count is usable
-    */
-    bool store_ubjson_count(const std::size_t count, std::size_t& result)
-    {
-        if (JSON_HEDLEY_UNLIKELY(count == npos))
-        {
-            return out_of_range_error(exception_id::container_too_large, "integer value overflow", "size");
-        }
-        result = count;
-        return true;
-    }
-
     /*!
     @brief read a UBJSON/BJData optimized-container count of a signed marker
            type ('i', 'I', 'l', 'L') and narrow it to std::size_t
@@ -21199,7 +21176,8 @@ class binary_reader
         {
             return out_of_range_error(exception_id::container_too_large, "integer value overflow", "size");
         }
-        return store_ubjson_count(static_cast<std::size_t>(number), result); // NOLINT(bugprone-signed-char-misuse,cert-str34-c): number is not a char
+        result = static_cast<std::size_t>(number); // NOLINT(bugprone-signed-char-misuse,cert-str34-c): number is not a char
+        return true;
     }
 
     /*!
@@ -21276,7 +21254,8 @@ class binary_reader
                 {
                     return false;
                 }
-                return store_ubjson_count(conditional_static_cast<std::size_t>(number), result);
+                result = conditional_static_cast<std::size_t>(number);
+                return true;
             }
 
             case 'M':
@@ -21294,7 +21273,8 @@ class binary_reader
                 {
                     return out_of_range_error(exception_id::container_too_large, "integer value overflow", "size");
                 }
-                return store_ubjson_count(detail::conditional_static_cast<std::size_t>(number), result);
+                result = detail::conditional_static_cast<std::size_t>(number);
+                return true;
             }
 
             case '[':

single_include/nlohmann/json_view.hpp

@@ -5059,7 +5059,8 @@ void value_range(const document_data& d, const node* n, std::size_t& start, std:
 
 /// the context basic_json puts in front of the message of an exception about
 /// a value: "(/path) " with JSON_DIAGNOSTICS (none for the root), then
-/// "(bytes a-b) " with JSON_DIAGNOSTIC_POSITIONS (also for the root)
+/// "(bytes a-b) " with JSON_DIAGNOSTIC_POSITIONS (also for the root); with
+/// JSON_DIAGNOSTICS, nothing for a value that is no longer part of the document
 ///
 /// This and the throw helpers that take an error_site are templates, so that
 /// the path and range code is instantiated only where an exception can be
@@ -5080,8 +5081,14 @@ std::string diagnostics(const error_site site)
     std::string context;
 #if JSON_DIAGNOSTICS
     std::vector<std::string> tokens;
-    // a value that is not found leaves no tokens, like the root
-    static_cast<void>(edited ? value_path<true>(d, site.value(), tokens) : value_path<false>(d, site.value(), tokens));
+    // a value that is no longer part of the document has no path, and its
+    // node still holds the bytes it had in the source: no context at all
+    const bool found = edited ? value_path<true>(d, site.value(), tokens) : value_path<false>(d, site.value(), tokens);
+    if (!found)
+    {
+        // context is still empty; returning it keeps the copy elided (-Wnrvo)
+        return context;
+    }
     if (!tokens.empty())
     {
         // the root value has no path, but still has a byte range

tests/src/unit-32bit.cpp

@@ -97,23 +97,3 @@ TEST_CASE("CBOR")
         }
     }
 }
-
-TEST_CASE("MessagePack")
-{
-    SECTION("parse errors")
-    {
-        SECTION("array/map size equal to the unknown-size sentinel")
-        {
-            // on 32-bit platforms an array 32/map 32 length of 0xFFFFFFFF aliases unknown_size()
-            std::vector<uint8_t> const varr = {0xDD, 0xFF, 0xFF, 0xFF, 0xFF};
-            std::vector<uint8_t> const vmap = {0xDF, 0xFF, 0xFF, 0xFF, 0xFF};
-
-            json _;
-            CHECK_THROWS_WITH_AS(_ = json::from_msgpack(varr), "[json.exception.out_of_range.408] syntax error while parsing MessagePack size: excessive array size", json::out_of_range&);
-            CHECK(json::from_msgpack(varr, true, false).is_discarded());
-
-            CHECK_THROWS_WITH_AS(_ = json::from_msgpack(vmap), "[json.exception.out_of_range.408] syntax error while parsing MessagePack size: excessive map size", json::out_of_range&);
-            CHECK(json::from_msgpack(vmap, true, false).is_discarded());
-        }
-    }
-}

tests/src/unit-bjdata.cpp

@@ -3237,32 +3237,12 @@ TEST_CASE("BJData")
 #endif
                 CHECK(json::from_bjdata(vM, true, false).is_discarded());
 
-                // the first dimension is 2^64-1, which is rejected as a count (see below)
-                CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vMX), "[json.exception.out_of_range.408] syntax error while parsing BJData size: integer value overflow", json::out_of_range&);
-                CHECK(json::from_bjdata(vMX, true, false).is_discarded());
-            }
-
-            SECTION("optimized array: count equal to SIZE_MAX")
-            {
-                // the reader marks a container without a count by SIZE_MAX, so
-                // a count of that value must not be read as "no count"
-                // (it used to parse as [1])
-                json _;
 #if SIZE_MAX != 0xffffffff
-                std::vector<uint8_t> const vM = {'[', '#', 'M', 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 'i', 0x01, ']'};
-                CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vM), "[json.exception.out_of_range.408] syntax error while parsing BJData size: integer value overflow", json::out_of_range&);
-                CHECK(json::from_bjdata(vM, true, false).is_discarded());
-                // one less is an ordinary excessive count
-                std::vector<uint8_t> const vM1 = {'[', '#', 'M', 0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 'i', 0x01, ']'};
-                CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vM1), "[json.exception.out_of_range.408] excessive array size: 18446744073709551614", json::out_of_range&);
+                CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vMX), "[json.exception.out_of_range.408] syntax error while parsing BJData size: excessive ndarray size caused overflow", json::out_of_range&);
 #else
-                std::vector<uint8_t> const vm = {'[', '#', 'm', 0xFF, 0xFF, 0xFF, 0xFF, 'i', 0x01, ']'};
-                CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vm), "[json.exception.out_of_range.408] syntax error while parsing BJData size: integer value overflow", json::out_of_range&);
-                CHECK(json::from_bjdata(vm, true, false).is_discarded());
-                std::vector<uint8_t> const vL = {'[', '#', 'L', 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 'i', 0x01, ']'};
-                CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vL), "[json.exception.out_of_range.408] syntax error while parsing BJData size: integer value overflow", json::out_of_range&);
-                CHECK(json::from_bjdata(vL, true, false).is_discarded());
+                CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vMX), "[json.exception.out_of_range.408] syntax error while parsing BJData size: integer value overflow", json::out_of_range&);
 #endif
+                CHECK(json::from_bjdata(vMX, true, false).is_discarded());
             }
 
             SECTION("optimized array: integer value overflow")

tests/src/unit-regression2.cpp

@@ -1009,7 +1009,7 @@ TEST_CASE("regression test - excessive binary container size honors allow_except
     // allow_exceptions=true (the default) must still throw exactly as before.
     // The exact message text is not checked here: on platforms where
     // std::size_t is 32-bit, the CBOR reader's own length-narrowing check
-    // (get_container_size(), unrelated to this fix) intercepts a
+    // (get_cbor_container_size(), unrelated to this fix) intercepts a
     // declared length of 2^63 before it ever reaches the check this test
     // targets, with different (but equally valid, and already correct)
     // wording -- see unit-cbor.cpp for coverage of that message.

tests/src/unit-ubjson.cpp

@@ -2385,18 +2385,6 @@ TEST_CASE("UBJSON optimized arrays of a valueless type are bounded")
         }
     }
 
-    SECTION("a count equal to SIZE_MAX is rejected")
-    {
-        // the reader marks a container without a count by SIZE_MAX, so a
-        // count of that value must not be read as "no count"; UBJSON's
-        // largest count type (int64) can only reach it where size_t is 32 bits
-#if SIZE_MAX == 0xffffffff
-        const std::vector<uint8_t> input = {'[', '#', 'L', 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, 'i', 0x01, ']'};
-        CHECK_THROWS_WITH_AS(_ = json::from_ubjson(input), "[json.exception.out_of_range.408] syntax error while parsing UBJSON size: integer value overflow", json::out_of_range&);
-        CHECK(json::from_ubjson(input, true, false).is_discarded());
-#endif
-    }
-
     SECTION("ordinary counts are unaffected")
     {
         CHECK(json::from_ubjson(std::vector<uint8_t>({'[', '$', 'Z', '#', 'i', 3})) == json({nullptr, nullptr, nullptr}));