Fix CBOR tag handling in cbor_tag_handler_t::store for non-binary items (#5559)

When using cbor_tag_handler_t::store, tags 0xD8-0xDB previously assumed
that the tagged item was a byte string, unconditionally attempting to
parse binary data and failing on valid CBOR documents containing tags
applied to integers, strings, arrays, or objects (such as self-describe
tag 55799).

Check whether the tagged data item is a byte string (0x40-0x5B or 0x5F).
If it is a byte string, store the subtype on the binary value as before.
Otherwise, iteratively process the tagged value in the driver loop using
item_read so that chained tags do not consume native stack space.

Part of #5316.

Signed-off-by: ReturnKartikey <kartikeynegi2000.work@gmail.com>

docs/mkdocs/docs/api/basic_json/cbor_tag_handler_t.md

@@ -18,7 +18,7 @@ ignore
 :   ignore tags
 
 store
-:   store tagged values as binary container with subtype (for bytes 0xd8..0xdb)
+:   store tagged byte strings (for bytes 0xd8..0xdb) as binary values with the tag as subtype; other tagged values are read as if the tag were ignored. If several tags precede a byte string, only the innermost one is stored.
 
 ## Examples
 

docs/mkdocs/docs/features/binary_formats/cbor.md

@@ -188,7 +188,7 @@ The library maps CBOR types to JSON value types as follows:
 
 !!! warning "Tagged items"
 
-    Tagged items (0xC0..0xDB) will throw a parse error by default. They can be ignored by passing `cbor_tag_handler_t::ignore` to function `from_cbor`, in which case the tag is skipped and the enclosed data item is parsed on its own. They can be stored by passing `cbor_tag_handler_t::store` to function `from_cbor`. Note that no tag is ever interpreted: for instance, a text string tagged with tag 0 (date/time) stays a string.
+    Tagged items (0xC0..0xDB) will throw a parse error by default. They can be ignored by passing `cbor_tag_handler_t::ignore` to function `from_cbor`, in which case the tag is skipped and the enclosed data item is parsed on its own. Passing `cbor_tag_handler_t::store` to function `from_cbor` stores tagged byte strings (for bytes 0xd8..0xdb) as binary values with the tag as subtype; other tagged values are read as if the tag were ignored. If several tags precede a byte string, only the innermost one is stored. Note that no tag is ever interpreted: for instance, a text string tagged with tag 0 (date/time) stays a string.
 
 ??? example
 

include/nlohmann/detail/input/binary_reader.hpp

@@ -44,7 +44,7 @@ enum class cbor_tag_handler_t
 {
     error,   ///< throw a parse_error exception in case of a tag
     ignore,  ///< ignore tags
-    store    ///< store tags as binary type
+    store    ///< store tagged byte strings (for bytes 0xd8..0xdb) as binary values with the tag as subtype; other tagged values are read as if the tag were ignored
 };
 
 /*!
@@ -592,14 +592,18 @@ class binary_reader
                          input (true) or whether the last read character should
                          be considered instead (false)
     @param[in] tag_handler how CBOR tags should be treated
+    @param[out] tag_pending whether a tag was parsed and its value follows
+    @param[out] item_read whether the tagged value's initial byte is already in current
 
     @return whether a valid CBOR value was passed to the SAX parser
     */
     bool parse_cbor_value(const bool get_char,
                           const cbor_tag_handler_t tag_handler,
-                          bool& tag_pending)
+                          bool& tag_pending,
+                          bool& item_read)
     {
         tag_pending = false;
+        item_read = false;
 
         switch (get_char ? get() : current)
         {
@@ -1021,7 +1025,17 @@ class binary_reader
                             }
                         }
                         get();
-                        return get_cbor_binary(b) && sax->binary(b);
+                        // a byte string (the heads accepted by get_cbor_binary) keeps the tag as subtype
+                        if ((current >= 0x40 && current <= 0x5B) || current == 0x5F)
+                        {
+                            return get_cbor_binary(b) && sax->binary(b);
+                        }
+
+                        // not a byte string: the tagged value, whose first byte
+                        // was just read, is read by the caller like for ignore
+                        tag_pending = true;
+                        item_read = true;
+                        return true;
                     }
 
                     default:                 // LCOV_EXCL_LINE
@@ -1503,13 +1517,14 @@ class binary_reader
 
             // a tag is not a value of its own: read on until the tagged value
             bool tag_pending = false;
+            bool item_read = false;
             do
             {
-                if (JSON_HEDLEY_UNLIKELY(!parse_cbor_value(fetch, tag_handler, tag_pending)))
+                if (JSON_HEDLEY_UNLIKELY(!parse_cbor_value(fetch, tag_handler, tag_pending, item_read)))
                 {
                     return false;
                 }
-                fetch = true;
+                fetch = !item_read;
             }
             while (tag_pending);
 

single_include/nlohmann/json.hpp

@@ -12741,7 +12741,7 @@ enum class cbor_tag_handler_t
 {
     error,   ///< throw a parse_error exception in case of a tag
     ignore,  ///< ignore tags
-    store    ///< store tags as binary type
+    store    ///< store tagged byte strings (for bytes 0xd8..0xdb) as binary values with the tag as subtype; other tagged values are read as if the tag were ignored
 };
 
 /*!
@@ -13289,14 +13289,18 @@ class binary_reader
                          input (true) or whether the last read character should
                          be considered instead (false)
     @param[in] tag_handler how CBOR tags should be treated
+    @param[out] tag_pending whether a tag was parsed and its value follows
+    @param[out] item_read whether the tagged value's initial byte is already in current
 
     @return whether a valid CBOR value was passed to the SAX parser
     */
     bool parse_cbor_value(const bool get_char,
                           const cbor_tag_handler_t tag_handler,
-                          bool& tag_pending)
+                          bool& tag_pending,
+                          bool& item_read)
     {
         tag_pending = false;
+        item_read = false;
 
         switch (get_char ? get() : current)
         {
@@ -13718,7 +13722,17 @@ class binary_reader
                             }
                         }
                         get();
-                        return get_cbor_binary(b) && sax->binary(b);
+                        // a byte string (the heads accepted by get_cbor_binary) keeps the tag as subtype
+                        if ((current >= 0x40 && current <= 0x5B) || current == 0x5F)
+                        {
+                            return get_cbor_binary(b) && sax->binary(b);
+                        }
+
+                        // not a byte string: the tagged value, whose first byte
+                        // was just read, is read by the caller like for ignore
+                        tag_pending = true;
+                        item_read = true;
+                        return true;
                     }
 
                     default:                 // LCOV_EXCL_LINE
@@ -14200,13 +14214,14 @@ class binary_reader
 
             // a tag is not a value of its own: read on until the tagged value
             bool tag_pending = false;
+            bool item_read = false;
             do
             {
-                if (JSON_HEDLEY_UNLIKELY(!parse_cbor_value(fetch, tag_handler, tag_pending)))
+                if (JSON_HEDLEY_UNLIKELY(!parse_cbor_value(fetch, tag_handler, tag_pending, item_read)))
                 {
                     return false;
                 }
-                fetch = true;
+                fetch = !item_read;
             }
             while (tag_pending);
 

tests/src/unit-cbor.cpp

@@ -2124,6 +2124,20 @@ TEST_CASE("CBOR nesting does not consume the call stack")
         CHECK(json::from_cbor(input, true, false, json::cbor_tag_handler_t::ignore).is_discarded());
     }
 
+    SECTION("stored tags")
+    {
+        // a tag over something other than a byte string is read like for
+        // ignore, so a chain of them must not recurse either (#5316)
+        std::vector<uint8_t> input;
+        for (std::size_t i = 0; i < 500000; ++i)
+        {
+            input.push_back(0xD8);
+            input.push_back(0x18);
+        }
+        input.push_back(0x01);
+        CHECK(json::from_cbor(input, true, true, json::cbor_tag_handler_t::store) == 1);
+    }
+
     SECTION("a well-formed deep value is read through the SAX interface")
     {
         std::vector<uint8_t> input(200000, 0x9F);
@@ -3054,6 +3068,77 @@ TEST_CASE("Tagged values")
             CHECK_THROWS_AS(_ = json::from_cbor(v_tagged, true, true, json::cbor_tag_handler_t::error), json::parse_error);
             CHECK_THROWS_AS(_ = json::from_cbor(v_tagged, true, true, json::cbor_tag_handler_t::ignore), json::parse_error);
         }
+
+        SECTION("issue #5316 - cbor_tag_handler_t::store on non-binary tagged items")
+        {
+            // 55799({"a": 1}) -- CBOR self-describe magic followed by a map
+            const std::vector<std::uint8_t> v_map{0xD9, 0xD9, 0xF7, 0xA1, 0x61, 0x61, 0x01};
+            CHECK(json::from_cbor(v_map, true, true, json::cbor_tag_handler_t::ignore) == json({{"a", 1}}));
+            CHECK(json::from_cbor(v_map, true, true, json::cbor_tag_handler_t::store) == json({{"a", 1}}));
+
+            // Tag 24 over unsigned integer 5
+            const std::vector<std::uint8_t> v_int{0xD8, 0x18, 0x05};
+            CHECK(json::from_cbor(v_int, true, true, json::cbor_tag_handler_t::ignore) == 5);
+            CHECK(json::from_cbor(v_int, true, true, json::cbor_tag_handler_t::store) == 5);
+
+            // Tag 24 over text string "foo"
+            const std::vector<std::uint8_t> v_str{0xD8, 0x18, 0x63, 'f', 'o', 'o'};
+            CHECK(json::from_cbor(v_str, true, true, json::cbor_tag_handler_t::ignore) == "foo");
+            CHECK(json::from_cbor(v_str, true, true, json::cbor_tag_handler_t::store) == "foo");
+
+            // Tag 24 over array [1, 2]
+            const std::vector<std::uint8_t> v_arr{0xD8, 0x18, 0x82, 0x01, 0x02};
+            CHECK(json::from_cbor(v_arr, true, true, json::cbor_tag_handler_t::ignore) == json({1, 2}));
+            CHECK(json::from_cbor(v_arr, true, true, json::cbor_tag_handler_t::store) == json({1, 2}));
+
+            // Tag 24 over boolean true
+            const std::vector<std::uint8_t> v_bool{0xD8, 0x18, 0xF5};
+            CHECK(json::from_cbor(v_bool, true, true, json::cbor_tag_handler_t::ignore) == true);
+            CHECK(json::from_cbor(v_bool, true, true, json::cbor_tag_handler_t::store) == true);
+
+            // Tag 24 over null
+            const std::vector<std::uint8_t> v_null{0xD8, 0x18, 0xF6};
+            CHECK(json::from_cbor(v_null, true, true, json::cbor_tag_handler_t::ignore) == nullptr);
+            CHECK(json::from_cbor(v_null, true, true, json::cbor_tag_handler_t::store) == nullptr);
+
+            // Nested tags: tag 55799 over tag 24 over integer 42
+            const std::vector<std::uint8_t> v_nested{0xD9, 0xD9, 0xF7, 0xD8, 0x18, 0x18, 0x2A};
+            CHECK(json::from_cbor(v_nested, true, true, json::cbor_tag_handler_t::ignore) == 42);
+            CHECK(json::from_cbor(v_nested, true, true, json::cbor_tag_handler_t::store) == 42);
+
+            // Tag 24 over byte string continues to store subtype as before
+            const std::vector<std::uint8_t> v_bin{0xD8, 0x18, 0x42, 0xCA, 0xFE};
+            auto j_bin_store = json::from_cbor(v_bin, true, true, json::cbor_tag_handler_t::store);
+            CHECK(j_bin_store.is_binary());
+            CHECK(j_bin_store.get_binary().has_subtype());
+            CHECK(j_bin_store.get_binary().subtype() == 24);
+            CHECK(j_bin_store.get_binary() == json::binary({0xCA, 0xFE}, 24).get_binary());
+
+            // Tagged values inside a container under store: [24(1), 25(h'0001')]
+            const std::vector<std::uint8_t> v_container{0x82, 0xD8, 0x18, 0x01, 0xD8, 0x19, 0x42, 0x00, 0x01};
+            auto j_container_store = json::from_cbor(v_container, true, true, json::cbor_tag_handler_t::store);
+            CHECK(j_container_store.is_array());
+            CHECK(j_container_store.size() == 2);
+            CHECK(j_container_store[0] == 1);
+            CHECK(j_container_store[1].is_binary());
+            CHECK(j_container_store[1].get_binary().has_subtype());
+            CHECK(j_container_store[1].get_binary().subtype() == 25);
+            CHECK(j_container_store[1].get_binary() == json::binary({0x00, 0x01}, 25).get_binary());
+
+            // Tagged values as object values under store: {"a": 55799(1), "b": 24(h'01')}
+            const std::vector<std::uint8_t> v_object{0xA2, 0x61, 'a', 0xD9, 0xD9, 0xF7, 0x01, 0x61, 'b', 0xD8, 0x18, 0x41, 0x01};
+            CHECK(json::from_cbor(v_object, true, true, json::cbor_tag_handler_t::store) == json({{"a", 1}, {"b", json::binary({0x01}, 24)}}));
+
+            // two tags in a row before a byte string: the inner tag is stored
+            // (this uses item_read and then the byte-string path)
+            const std::vector<std::uint8_t> v_nested_byte_string{0xD8, 0x18, 0xD8, 0x19, 0x42, 0x00, 0x01};
+            CHECK(json::from_cbor(v_nested_byte_string, true, true, json::cbor_tag_handler_t::store) == json::binary({0x00, 0x01}, 25));
+
+            // errors after a stored tag are now the same as with ignore
+            json _;
+            CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector<std::uint8_t> {0xD8, 0x18}, true, true, json::cbor_tag_handler_t::store), "[json.exception.parse_error.110] parse error at byte 3: syntax error while parsing CBOR value: unexpected end of input", json::parse_error&);
+            CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector<std::uint8_t> {0xD8, 0x18, 0x1C}, true, true, json::cbor_tag_handler_t::store), "[json.exception.parse_error.112] parse error at byte 3: syntax error while parsing CBOR value: invalid byte: 0x1C", json::parse_error&);
+        }
     }
 
     SECTION("negative integer overflow")