From 93c0a341715767384fe91750507dad51b28abc42 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Wed, 23 Sep 2026 20:42:07 +0200 Subject: [PATCH] :zap: validate only newly read bytes of binary-format strings get_string() validated the whole result after each call, but get_bytes() appends to it and CBOR indefinite-length strings collect all chunks in the same result, so every chunk re-validated everything read before it. An input of many small chunks took quadratic time (80000 one-byte chunks, 160 KB of input, took about 7 seconds). Only the newly read bytes are validated now, which also matches RFC 8949's requirement that every chunk is valid UTF-8 on its own. Signed-off-by: Niels Lohmann --- .../nlohmann/detail/input/binary_reader.hpp | 6 +++- include/nlohmann/detail/string_utils.hpp | 11 ++++--- single_include/nlohmann/json.hpp | 17 +++++++--- tests/src/unit-cbor.cpp | 32 +++++++++++++++++++ 4 files changed, 56 insertions(+), 10 deletions(-) diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index 7a77edd7e..2940f2a96 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -3314,6 +3314,10 @@ class binary_reader const NumberType len, string_t& result) { + // get_bytes() appends to result, and CBOR indefinite-length strings + // collect all their chunks in the same result; validating only the + // newly read bytes keeps the check linear in the input size + const std::size_t old_size = result.size(); if (JSON_HEDLEY_UNLIKELY(!get_bytes(format, len, "string", result))) { return false; @@ -3324,7 +3328,7 @@ class binary_reader // right here so malformed input is caught at decode time instead of // only surfacing later as a type_error.316 when the value is dumped // (which would defeat allow_exceptions=false / strict discarding). - if (JSON_HEDLEY_UNLIKELY(!is_valid_utf8(result))) + if (JSON_HEDLEY_UNLIKELY(!is_valid_utf8(result, old_size))) { return sax->parse_error(chars_read, get_token_string(), parse_error::create(113, chars_read, diff --git a/include/nlohmann/detail/string_utils.hpp b/include/nlohmann/detail/string_utils.hpp index 064d5ccf9..142943cd6 100644 --- a/include/nlohmann/detail/string_utils.hpp +++ b/include/nlohmann/detail/string_utils.hpp @@ -109,16 +109,19 @@ MessagePack/BSON specifications all require text strings to be UTF-8), so that malformed input is caught immediately instead of only surfacing later as a type_error.316 when the resulting value is dumped. -@param[in] s the string to check -@return whether @a s is valid UTF-8 +@param[in] s the string to check +@param[in] first index of the first byte to check; the bytes before it are + assumed to have been validated already and to end on a + code point boundary +@return whether @a s (from index @a first on) is valid UTF-8 */ template -inline bool is_valid_utf8(const StringType& s) noexcept +inline bool is_valid_utf8(const StringType& s, const std::size_t first = 0) noexcept { std::uint8_t state = UTF8_ACCEPT; std::uint32_t codepoint = 0; - for (std::size_t i = 0; i < s.size(); ++i) + for (std::size_t i = first; i < s.size(); ++i) { decode(state, codepoint, static_cast(s[i])); if (state == UTF8_REJECT) diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index fa9ec890c..36574ee8d 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -6105,16 +6105,19 @@ MessagePack/BSON specifications all require text strings to be UTF-8), so that malformed input is caught immediately instead of only surfacing later as a type_error.316 when the resulting value is dumped. -@param[in] s the string to check -@return whether @a s is valid UTF-8 +@param[in] s the string to check +@param[in] first index of the first byte to check; the bytes before it are + assumed to have been validated already and to end on a + code point boundary +@return whether @a s (from index @a first on) is valid UTF-8 */ template -inline bool is_valid_utf8(const StringType& s) noexcept +inline bool is_valid_utf8(const StringType& s, const std::size_t first = 0) noexcept { std::uint8_t state = UTF8_ACCEPT; std::uint32_t codepoint = 0; - for (std::size_t i = 0; i < s.size(); ++i) + for (std::size_t i = first; i < s.size(); ++i) { decode(state, codepoint, static_cast(s[i])); if (state == UTF8_REJECT) @@ -15398,6 +15401,10 @@ class binary_reader const NumberType len, string_t& result) { + // get_bytes() appends to result, and CBOR indefinite-length strings + // collect all their chunks in the same result; validating only the + // newly read bytes keeps the check linear in the input size + const std::size_t old_size = result.size(); if (JSON_HEDLEY_UNLIKELY(!get_bytes(format, len, "string", result))) { return false; @@ -15408,7 +15415,7 @@ class binary_reader // right here so malformed input is caught at decode time instead of // only surfacing later as a type_error.316 when the value is dumped // (which would defeat allow_exceptions=false / strict discarding). - if (JSON_HEDLEY_UNLIKELY(!is_valid_utf8(result))) + if (JSON_HEDLEY_UNLIKELY(!is_valid_utf8(result, old_size))) { return sax->parse_error(chars_read, get_token_string(), parse_error::create(113, chars_read, diff --git a/tests/src/unit-cbor.cpp b/tests/src/unit-cbor.cpp index fc7fc614a..9c799b0eb 100644 --- a/tests/src/unit-cbor.cpp +++ b/tests/src/unit-cbor.cpp @@ -1854,6 +1854,38 @@ TEST_CASE("CBOR") CHECK(json::from_cbor(json::to_cbor(j)) == j); } + SECTION("invalid UTF-8 in indefinite-length string") + { + json _; + + // every chunk must be valid UTF-8 on its own (RFC 8949, Section + // 3.2.3), so a code point split across two chunks is rejected + CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector({0x7f, 0x61, 0xc3, 0x61, 0xa9, 0xff})), "[json.exception.parse_error.113] parse error at byte 3: syntax error while parsing CBOR string: invalid string: ill-formed UTF-8 byte", json::parse_error&); + CHECK(json::from_cbor(std::vector({0x7f, 0x61, 0xc3, 0x61, 0xa9, 0xff}), true, false).is_discarded()); + + // an ill-formed later chunk is rejected after valid ones + CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector({0x7f, 0x62, 0xc3, 0xa9, 0x62, 0xc0, 0xae, 0xff})), "[json.exception.parse_error.113] parse error at byte 7: syntax error while parsing CBOR string: invalid string: ill-formed UTF-8 byte", json::parse_error&); + + // valid multi-byte chunks are accepted + CHECK(json::from_cbor(std::vector({0x7f, 0x62, 0xc3, 0xa9, 0x62, 0xc3, 0xb6, 0xff})) == "\xc3\xa9\xc3\xb6"); + } + + SECTION("many chunks in indefinite-length string") + { + // only the newly read chunk is validated, not the whole string + // collected so far; validating the latter made this input take + // quadratic time (about ten seconds for 100000 chunks) + constexpr std::size_t chunks = 100000; + std::vector v{0x7f}; + for (std::size_t i = 0; i < chunks; ++i) + { + v.push_back(0x61); + v.push_back('a'); + } + v.push_back(0xff); + CHECK(json::from_cbor(v) == std::string(chunks, 'a')); + } + SECTION("strict mode") { std::vector const vec = {0xf6, 0xf6};