From 95169a63fa86662685980fb02495408810bd6a8e Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 25 Oct 2026 17:34:17 +0100 Subject: [PATCH] Skip CBOR tags in front of map keys and validate indefinite text chunks A tag in front of a text string map key was rejected with parse_error.113 even with cbor_tag_handler_t::ignore or store, although tags in front of values are handled. The tags are now skipped iteratively before the key is read; the handler error still rejects them. The chunks of an indefinite-length text string were concatenated before the single UTF-8 check, so a code point split over two chunks, or an invalid chunk, was accepted. RFC 8949, Section 3.2.3 requires every chunk to be valid on its own. With error_handler_t::strict, replace and ignore each chunk is now checked; keep is unchanged. The chunks are appended with append(ptr, len), so StringType needs nothing new. Signed-off-by: Niels Lohmann --- .../nlohmann/detail/input/binary_reader.hpp | 80 +++++++++- tests/src/unit-cbor.cpp | 145 +++++++++++++++++- 2 files changed, 214 insertions(+), 11 deletions(-) diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index d3c3d5bdc..b751e3c59 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -1218,6 +1218,10 @@ class binary_reader // level as required by RFC 8949, Section 3.2.3 bool indefinite = false; + // the chunk currently being validated; only used for the chunks of an + // indefinite-length string when the UTF-8 check can do something + string_t chunk; + while (true) { if (JSON_HEDLEY_UNLIKELY(!unexpect_eof("string"))) @@ -1237,26 +1241,82 @@ class binary_reader } // a break marker closes the indefinite-length string; outside - // of one it falls through to the error below + // of one it falls through to the error below. Every chunk has + // been checked on its own, so there is nothing left to check. if (indefinite && current == 0xFF) { - return check_string_utf8(result, context); - } - - if (JSON_HEDLEY_UNLIKELY(!get_cbor_string_chunk(result, indefinite))) - { - return false; + return true; } if (!indefinite) { - return check_string_utf8(result, context); + return get_cbor_string_chunk(result, false) && check_string_utf8(result, context); + } + + if (error_handler == error_handler_t::keep) + { + // nothing to check: read the chunk directly into the result + if (JSON_HEDLEY_UNLIKELY(!get_cbor_string_chunk(result, true))) + { + return false; + } + } + else + { + // RFC 8949, Section 3.2.3: each chunk is a text string of its + // own and must be valid UTF-8 by itself, i.e., a code point + // cannot be spread over two chunks + chunk.clear(); + if (JSON_HEDLEY_UNLIKELY(!get_cbor_string_chunk(chunk, true) || !check_string_utf8(chunk, context))) + { + return false; + } + result.append(chunk.data(), chunk.size()); } get(); } } + /*! + @brief skips the tags in front of a CBOR object key + + A tag may wrap any data item, so it may also wrap a key. Like in front of a + value, the tags are skipped for the handlers ignore and store; with the + handler error they are left to @ref get_cbor_object_key, which rejects the + key. The tag loop is iterative, so the number of tags is not limited by the + recursion depth. + + @param[in] tag_handler how CBOR tags should be treated + + @return whether the tags could be skipped (false on EOF in a tag argument) + + @pre @a current is the first byte of the key + */ + bool skip_cbor_key_tags(const cbor_tag_handler_t tag_handler) + { + if (tag_handler == cbor_tag_handler_t::error) + { + return true; + } + + // 0xC0..0xDB: tag with the tag number in the head (0xC0..0xD7) or in + // the 1, 2, 4, or 8 bytes that follow (0xD8..0xDB) + while (current >= 0xC0 && current <= 0xDB) + { + if (current >= 0xD8) + { + std::uint64_t tag_to_ignore{}; + if (JSON_HEDLEY_UNLIKELY(!get_cbor_argument(tag_to_ignore))) + { + return false; + } + } + get(); + } + return true; + } + /*! @brief reads a CBOR object key @@ -1623,6 +1683,10 @@ class binary_reader if (top.is_object) { + if (JSON_HEDLEY_UNLIKELY(!skip_cbor_key_tags(tag_handler))) + { + return false; + } if (JSON_HEDLEY_UNLIKELY(!read_object_key<&binary_reader::get_cbor_object_key>(key))) { return false; diff --git a/tests/src/unit-cbor.cpp b/tests/src/unit-cbor.cpp index 9ee371495..65d76bc8f 100644 --- a/tests/src/unit-cbor.cpp +++ b/tests/src/unit-cbor.cpp @@ -1807,6 +1807,93 @@ TEST_CASE("CBOR") CHECK(json::from_cbor(std::vector({0xa1, 0xff, 0x01}), true, false).is_discarded()); } + SECTION("tag in front of a key") + { + const auto ignore = json::cbor_tag_handler_t::ignore; + const auto store = json::cbor_tag_handler_t::store; + const json expected = {{"a", 10}}; + + // the tags are skipped in front of a text string key, like in + // front of a value + const std::vector> tagged = + { + {0xA1, 0xC0, 0x61, 0x61, 0x0A}, // tag in the head + {0xA1, 0xD8, 0x05, 0x61, 0x61, 0x0A}, // 1-byte tag number + {0xA1, 0xD9, 0x01, 0x2C, 0x61, 0x61, 0x0A}, // 2-byte tag number + {0xA1, 0xDA, 0x00, 0x01, 0x00, 0x00, 0x61, 0x61, 0x0A}, // 4-byte tag number + {0xA1, 0xDB, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x61, 0x61, 0x0A}, // 8-byte tag number + {0xA1, 0xC0, 0xD8, 0x05, 0xC1, 0x61, 0x61, 0x0A}, // chained tags + {0xBF, 0xC0, 0x61, 0x61, 0x0A, 0xFF}, // indefinite-length map + {0xBF, 0x61, 0x62, 0x01, 0xC0, 0x61, 0x61, 0x0A, 0xFF}, // second key tagged + {0xA1, 0xC0, 0x7F, 0x61, 0x61, 0xFF, 0x0A}, // indefinite-length text key + {0xA1, 0xC0, 0x61, 0x61, 0xC0, 0x0A}, // tagged key and value + }; + + for (const auto& input : tagged) + { + CAPTURE(input) + json const j_ignore = json::from_cbor(input, true, true, ignore); + json const j_store = json::from_cbor(input, true, true, store); + CHECK(j_ignore.at("a") == 10); + CHECK(j_store.at("a") == 10); + CHECK(j_ignore == j_store); + } + CHECK(json::from_cbor(tagged[0], true, true, ignore) == expected); + CHECK(json::from_cbor(tagged[1], true, true, store) == expected); + + // nested maps + CHECK(json::from_cbor(std::vector({0xA1, 0xC0, 0x61, 0x61, 0xA1, 0xD8, 0x05, 0x61, 0x62, 0x01}), true, true, ignore) == json{{"a", {{"b", 1}}}}); + + // the tag handler error still rejects every tag + for (const auto& input : tagged) + { + CAPTURE(input) + json _; + CHECK_THROWS_AS(_ = json::from_cbor(input), json::parse_error&); + } + { + json _; + CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector({0xA1, 0xD8, 0x05, 0x61, 0x61, 0x0A})), "[json.exception.parse_error.113] parse error at byte 2: syntax error while parsing CBOR object key: only string keys are supported, but found a tag; last byte: 0xD8", json::parse_error&); + } + + // a tag does not turn a key of another type into a text string + { + json _; + CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector({0xA1, 0xC0, 0x01, 0x01}), true, true, ignore), "[json.exception.parse_error.113] parse error at byte 3: syntax error while parsing CBOR object key: only string keys are supported, but found an unsigned integer; last byte: 0x01", json::parse_error&); + CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector({0xA1, 0xD8, 0x05, 0x41, 0x61, 0x01}), true, true, store), "[json.exception.parse_error.113] parse error at byte 4: syntax error while parsing CBOR object key: only string keys are supported, but found a byte string; last byte: 0x41", json::parse_error&); + CHECK(json::from_cbor(std::vector({0xA1, 0xC0, 0x01, 0x01}), true, false, ignore).is_discarded()); + } + + // end of input after the tag, or inside the tag number + { + json _; + CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector({0xA1, 0xC0}), true, true, ignore), "[json.exception.parse_error.110] parse error at byte 3: syntax error while parsing CBOR string: unexpected end of input", json::parse_error&); + CHECK_THROWS_AS(_ = json::from_cbor(std::vector({0xA1, 0xD8}), true, true, ignore), json::parse_error&); + CHECK_THROWS_AS(_ = json::from_cbor(std::vector({0xA1, 0xDB, 0x00, 0x00}), true, true, store), json::parse_error&); + CHECK_THROWS_AS(_ = json::from_cbor(std::vector({0xBF, 0xC0}), true, true, store), json::parse_error&); + } + + // SAX: the key event is reported without the tag (start_object, + // key, number_unsigned, end_object are four events) + { + const std::vector input = {0xA1, 0xC0, 0x61, 0x61, 0x0A}; + SaxCountdown four(4); + CHECK(json::sax_parse(input, &four, json::input_format_t::cbor, true, false, false, ignore)); + SaxCountdown three(3); + CHECK(!json::sax_parse(input, &three, json::input_format_t::cbor, true, false, false, ignore)); + SaxCountdown error_handler(4); + CHECK(!json::sax_parse(input, &error_handler, json::input_format_t::cbor)); + } + + // a long chain of tags needs no recursion + { + std::vector input = {0xA1}; + input.insert(input.end(), 100000, 0xC0); + input.insert(input.end(), {0x61, 0x61, 0x0A}); + CHECK(json::from_cbor(input, true, true, ignore) == expected); + } + } + SECTION("non-string key (see #2766 and #3381)") { // only text strings map to JSON object keys; any other key is @@ -1915,9 +2002,10 @@ TEST_CASE("CBOR") { json _; - // the chunks are concatenated as is, without checking that each - // chunk is valid UTF-8 on its own (RFC 8949, Section 3.2.3), so - // a code point split across two chunks yields a valid string + // with the default error handler keep, the chunks are concatenated + // as is, without checking that each chunk is valid UTF-8 on its + // own (RFC 8949, Section 3.2.3), so a code point split across two + // chunks yields a valid string CHECK_NOTHROW(_ = json::from_cbor(std::vector({0x7f, 0x61, 0xc3, 0x61, 0xa9, 0xff}))); CHECK(_ == "\xc3\xa9"); CHECK(_.dump() == "\"\xc3\xa9\""); @@ -1937,6 +2025,57 @@ TEST_CASE("CBOR") CHECK(json::from_cbor(std::vector({0x7f, 0x62, 0xc3, 0xa9, 0x62, 0xc3, 0xb6, 0xff})) == "\xc3\xa9\xc3\xb6"); } + SECTION("ill-formed UTF-8 in a chunk of an indefinite-length string (RFC 8949, Section 3.2.3)") + { + const auto strict = json::error_handler_t::strict; + const auto replace = json::error_handler_t::replace; + const auto ignore = json::error_handler_t::ignore; + const auto keep = json::error_handler_t::keep; + const auto tags = json::cbor_tag_handler_t::error; + + // a code point split over two chunks + const std::vector split = {0x7f, 0x61, 0xc3, 0x61, 0xa9, 0xff}; + // a 4-byte code point split over two chunks + const std::vector split_emoji = {0x7f, 0x62, 0xf0, 0x9f, 0x62, 0x98, 0x80, 0xff}; + // the same in an object key + const std::vector split_key = {0xa1, 0x7f, 0x61, 0xc3, 0x61, 0xa9, 0xff, 0x01}; + // a valid chunk followed by one with a truncated code point + const std::vector truncated = {0x7f, 0x62, 0x41, 0xc3, 0x61, 0xa9, 0xff}; + + // strict: the string is invalid + for (const auto& input : {split, split_emoji, split_key, truncated}) + { + CAPTURE(input) + json _; + CHECK_THROWS_AS(_ = json::from_cbor(input, true, true, tags, strict), json::parse_error&); + CHECK(json::from_cbor(input, true, false, tags, strict).is_discarded()); + } + { + json _; + CHECK_THROWS_WITH_AS(_ = json::from_cbor(split, true, true, tags, strict), "[json.exception.parse_error.113] parse error at byte 3: syntax error while parsing CBOR string: invalid string: ill-formed UTF-8 byte", json::parse_error&); + CHECK_THROWS_WITH_AS(_ = json::from_cbor(split_key, true, true, tags, strict), "[json.exception.parse_error.113] parse error at byte 4: syntax error while parsing CBOR key: invalid string: ill-formed UTF-8 byte", json::parse_error&); + } + + // replace and ignore: every chunk is sanitized on its own + CHECK(json::from_cbor(split, true, true, tags, replace) == "\xEF\xBF\xBD\xEF\xBF\xBD"); + CHECK(json::from_cbor(split, true, true, tags, ignore) == ""); + CHECK(json::from_cbor(truncated, true, true, tags, replace) == "A\xEF\xBF\xBD\xEF\xBF\xBD"); + CHECK(json::from_cbor(truncated, true, true, tags, ignore) == "A"); + CHECK(json::from_cbor(split_key, true, true, tags, replace) == json{{"\xEF\xBF\xBD\xEF\xBF\xBD", 1}}); + + // keep: unchanged, the chunks are concatenated as is + CHECK(json::from_cbor(split, true, true, tags, keep) == "\xc3\xa9"); + CHECK(json::from_cbor(split_emoji, true, true, tags, keep) == "\xf0\x9f\x98\x80"); + + // conforming input is unaffected by the error handler + const std::vector valid = {0x7f, 0x62, 0xc3, 0xa9, 0x64, 0xf0, 0x9f, 0x98, 0x80, 0x60, 0xff}; + for (const auto handler : {strict, replace, ignore, keep}) + { + CHECK(json::from_cbor(valid, true, true, tags, handler) == "\xc3\xa9\xf0\x9f\x98\x80"); + CHECK(json::from_cbor(std::vector({0x7f, 0xff}), true, true, tags, handler) == ""); + } + } + SECTION("many chunks in indefinite-length string") { constexpr std::size_t chunks = 100000;