Reject malformed UTF-16/UTF-32 units in wide-string input

The wide-string input adapter (used for std::u16string, std::u32string,
std::wstring, and iterators over 2- or 4-byte character types) passed
some malformed code units on to the lexer as values that are neither a
byte (0x00..0xFF) nor char_traits<char>::eof(). As a result:

- A lone UTF-16 surrogate inside true/false/null was accepted if its low
  byte matched the expected letter, or ended the input silently if it
  was the last unit.
- A high surrogate followed by a unit that is not its low surrogate
  swallowed that unit; if the swallowed unit was the newline ending a
  // comment, the comment silently extended over the next line.
- Where wint_t is a signed int (macOS, the BSDs), a negative wchar_t
  collided with char_traits<char>::eof() (ending the input early) or was
  truncated to its low byte, depending on its value.

The UTF-32 helper now converts the code unit to std::uint32_t before the
range checks, so a negative unit reaches the same "emit 0xFF" branch
already used for code points above U+10FFFF. The UTF-16 helper now
peeks at the next unit before consuming it, and emits 0xFF instead of
the raw surrogate when no valid pair is found, matching how ill-formed
UTF-8 bytes are rejected elsewhere in the lexer.

Fixes #5645.
Signed-off-by: Niels Lohmann <mail@nlohmann.me>

docs/mkdocs/docs/api/basic_json/from_bjdata.md

@@ -111,12 +111,3 @@ Linear in the size of the input.
 - Added in version 3.11.0.
 - Extended container support (1) to include types with lvalue-only ADL `begin`/`end` (matching `std::begin`/`std::end` semantics) in version 3.13.0.
 - Extended overload (2) to accept heterogeneous iterator+sentinel pairs (C++20 ranges support) in version 3.13.0.
-
-!!! warning "Deprecation"
-
-    - Overload (2) replaces calls to `from_bjdata` with a pointer and a length as first two parameters, which has been
-      deprecated in version 3.13.0. This overload will be removed in version 4.0.0. Please replace all calls like
-      `#!cpp from_bjdata(ptr, len, ...);` with `#!cpp from_bjdata(ptr, ptr+len, ...);`.
-
-    You should be warned by your compiler with a `-Wdeprecated-declarations` warning if you are using a deprecated
-    function.

docs/mkdocs/docs/api/basic_json/from_bon8.md

@@ -106,12 +106,3 @@ Linear in the size of the input.
 ## Version history
 
 - Added in version 3.13.0.
-
-!!! warning "Deprecation"
-
-    - Overload (2) replaces calls to `from_bon8` with a pointer and a length as first two parameters, which has been
-      deprecated in version 3.13.0. This overload will be removed in version 4.0.0. Please replace all calls like
-      `#!cpp from_bon8(ptr, len, ...);` with `#!cpp from_bon8(ptr, ptr+len, ...);`.
-
-    You should be warned by your compiler with a `-Wdeprecated-declarations` warning if you are using a deprecated
-    function.

include/nlohmann/detail/input/input_adapters.hpp

@@ -445,8 +445,10 @@ struct wide_string_input_helper<BaseInputAdapter, 4>
         }
         else
         {
-            // get the current character
-            const auto wc = input.get_character();
+            // get the current character; converted to an unsigned type so that
+            // a negative unit (wint_t is signed on some platforms) is not
+            // mistaken for an ASCII character or for EOF
+            const auto wc = static_cast<std::uint32_t>(input.get_character());
 
             // UTF-32 to UTF-8 encoding
             if (wc < 0x80)
@@ -541,9 +543,11 @@ struct wide_string_input_helper<BaseInputAdapter, 2>
                 bool valid_pair = false;
                 if (wc <= 0xDBFF && JSON_HEDLEY_UNLIKELY(!input.empty()))
                 {
-                    const auto wc2 = static_cast<unsigned int>(input.get_character());
+                    // only consume the next unit if it completes the pair
+                    const auto wc2 = static_cast<unsigned int>(*input.current);
                     if (0xDC00 <= wc2 && wc2 <= 0xDFFF)
                     {
+                        input.get_character();
                         const auto charcode = 0x10000u + (((static_cast<unsigned int>(wc) & 0x3FFu) << 10u) | (wc2 & 0x3FFu));
                         utf8_bytes[0] = static_cast<std::char_traits<char>::int_type>(0xF0u | (charcode >> 18u));
                         utf8_bytes[1] = static_cast<std::char_traits<char>::int_type>(0x80u | ((charcode >> 12u) & 0x3Fu));
@@ -556,7 +560,8 @@ struct wide_string_input_helper<BaseInputAdapter, 2>
 
                 if (!valid_pair)
                 {
-                    utf8_bytes[0] = static_cast<std::char_traits<char>::int_type>(wc);
+                    // emit a byte that is never valid UTF-8 (see the UTF-32 case)
+                    utf8_bytes[0] = 0xFF;
                     utf8_bytes_filled = 1;
                 }
             }

include/nlohmann/json.hpp

@@ -5547,16 +5547,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
         return result;
     }
 
-    template<typename T>
-    JSON_HEDLEY_WARN_UNUSED_RESULT
-    JSON_HEDLEY_DEPRECATED_FOR(3.13.0, from_bjdata(ptr, ptr + len))
-    static basic_json from_bjdata(const T* ptr, std::size_t len,
-                                  const bool strict = true,
-                                  const bool allow_exceptions = true)
-    {
-        return from_bjdata(ptr, ptr + len, strict, allow_exceptions);
-    }
-
     /// @brief create a JSON value from an input in BON8 format
     /// @sa https://json.nlohmann.me/api/basic_json/from_bon8/
     template<typename InputType>
@@ -5594,16 +5584,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
         return result;
     }
 
-    template<typename T>
-    JSON_HEDLEY_WARN_UNUSED_RESULT
-    JSON_HEDLEY_DEPRECATED_FOR(3.13.0, from_bon8(ptr, ptr + len))
-    static basic_json from_bon8(const T* ptr, std::size_t len,
-                                const bool strict = true,
-                                const bool allow_exceptions = true)
-    {
-        return from_bon8(ptr, ptr + len, strict, allow_exceptions);
-    }
-
     /// @brief create a JSON value from an input in BSON format
     /// @sa https://json.nlohmann.me/api/basic_json/from_bson/
     template<typename InputType>

single_include/nlohmann/json.hpp

@@ -7989,8 +7989,10 @@ struct wide_string_input_helper<BaseInputAdapter, 4>
         }
         else
         {
-            // get the current character
-            const auto wc = input.get_character();
+            // get the current character; converted to an unsigned type so that
+            // a negative unit (wint_t is signed on some platforms) is not
+            // mistaken for an ASCII character or for EOF
+            const auto wc = static_cast<std::uint32_t>(input.get_character());
 
             // UTF-32 to UTF-8 encoding
             if (wc < 0x80)
@@ -8085,9 +8087,11 @@ struct wide_string_input_helper<BaseInputAdapter, 2>
                 bool valid_pair = false;
                 if (wc <= 0xDBFF && JSON_HEDLEY_UNLIKELY(!input.empty()))
                 {
-                    const auto wc2 = static_cast<unsigned int>(input.get_character());
+                    // only consume the next unit if it completes the pair
+                    const auto wc2 = static_cast<unsigned int>(*input.current);
                     if (0xDC00 <= wc2 && wc2 <= 0xDFFF)
                     {
+                        input.get_character();
                         const auto charcode = 0x10000u + (((static_cast<unsigned int>(wc) & 0x3FFu) << 10u) | (wc2 & 0x3FFu));
                         utf8_bytes[0] = static_cast<std::char_traits<char>::int_type>(0xF0u | (charcode >> 18u));
                         utf8_bytes[1] = static_cast<std::char_traits<char>::int_type>(0x80u | ((charcode >> 12u) & 0x3Fu));
@@ -8100,7 +8104,8 @@ struct wide_string_input_helper<BaseInputAdapter, 2>
 
                 if (!valid_pair)
                 {
-                    utf8_bytes[0] = static_cast<std::char_traits<char>::int_type>(wc);
+                    // emit a byte that is never valid UTF-8 (see the UTF-32 case)
+                    utf8_bytes[0] = 0xFF;
                     utf8_bytes_filled = 1;
                 }
             }
@@ -31628,16 +31633,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
         return result;
     }
 
-    template<typename T>
-    JSON_HEDLEY_WARN_UNUSED_RESULT
-    JSON_HEDLEY_DEPRECATED_FOR(3.13.0, from_bjdata(ptr, ptr + len))
-    static basic_json from_bjdata(const T* ptr, std::size_t len,
-                                  const bool strict = true,
-                                  const bool allow_exceptions = true)
-    {
-        return from_bjdata(ptr, ptr + len, strict, allow_exceptions);
-    }
-
     /// @brief create a JSON value from an input in BON8 format
     /// @sa https://json.nlohmann.me/api/basic_json/from_bon8/
     template<typename InputType>
@@ -31675,16 +31670,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
         return result;
     }
 
-    template<typename T>
-    JSON_HEDLEY_WARN_UNUSED_RESULT
-    JSON_HEDLEY_DEPRECATED_FOR(3.13.0, from_bon8(ptr, ptr + len))
-    static basic_json from_bon8(const T* ptr, std::size_t len,
-                                const bool strict = true,
-                                const bool allow_exceptions = true)
-    {
-        return from_bon8(ptr, ptr + len, strict, allow_exceptions);
-    }
-
     /// @brief create a JSON value from an input in BSON format
     /// @sa https://json.nlohmann.me/api/basic_json/from_bson/
     template<typename InputType>

tests/src/unit-bjdata.cpp

@@ -4572,31 +4572,3 @@ TEST_CASE("BJData roundtrips" * doctest::skip())
         }
     }
 }
-
-TEST_CASE("issue #5648 - from_bjdata(ptr, len) must read len bytes, not treat ptr as a C string")
-{
-    // to_bjdata() encodes the integer 0 as the two bytes 'i' 0x00 (a BJData
-    // type marker followed by the value byte 0x00), so the packed data
-    // below contains a 0x00 byte before its end.
-    const json j = {{"a", 0}};
-    const std::vector<std::uint8_t> packed = json::to_bjdata(j);
-    bool contains_nul = false;
-    for (const auto byte : packed)
-    {
-        contains_nul |= (byte == 0x00);
-    }
-    REQUIRE(contains_nul);
-
-    // before the fix, from_bjdata had no (ptr, len) overload, so this call
-    // bound to from_bjdata(InputType&&, bool strict) instead: ptr was read
-    // as a NUL-terminated C string (stopping at the embedded 0x00 byte), and
-    // len was silently converted to the strict flag. The deprecated
-    // overload added for this issue forwards to from_bjdata(ptr, ptr + len,
-    // ...) instead, like from_ubjson's deprecated (ptr, len) overload does.
-    json result;
-    CHECK_NOTHROW(result = json::from_bjdata(packed.data(), packed.size()));
-    CHECK(result == j);
-
-    // len must not collapse into the strict flag either
-    CHECK(json::from_bjdata(packed.data(), packed.size(), false) == j);
-}

tests/src/unit-bon8.cpp

@@ -1010,34 +1010,6 @@ TEST_CASE("BON8 roundtrips" * doctest::skip())
     }
 }
 
-TEST_CASE("issue #5648 - from_bon8(ptr, len) must read len bytes, not treat ptr as a C string")
-{
-    // to_bon8() encodes the integer 40 as the two bytes 0xC2 0x00 (a BON8
-    // lead byte followed by a continuation byte of 0x00), so the packed data
-    // below contains a 0x00 byte before its end.
-    const json j = {{"a", 40}, {"b", "x"}};
-    const std::vector<std::uint8_t> packed = json::to_bon8(j);
-    bool contains_nul = false;
-    for (const auto byte : packed)
-    {
-        contains_nul |= (byte == 0x00);
-    }
-    REQUIRE(contains_nul);
-
-    // before the fix, from_bon8 had no (ptr, len) overload, so this call
-    // bound to from_bon8(InputType&&, bool strict) instead: ptr was read as
-    // a NUL-terminated C string (stopping at the embedded 0x00 byte), and
-    // len was silently converted to the strict flag. The deprecated
-    // overload added for this issue forwards to from_bon8(ptr, ptr + len,
-    // ...) instead, like from_cbor's deprecated (ptr, len) overload does.
-    json result;
-    CHECK_NOTHROW(result = json::from_bon8(packed.data(), packed.size()));
-    CHECK(result == j);
-
-    // len must not collapse into the strict flag either
-    CHECK(json::from_bon8(packed.data(), packed.size(), false) == j);
-}
-
 #ifdef JSON_HAS_CPP_17
 TEST_CASE("BON8 with std::byte")
 {

tests/src/unit-wstring.cpp

@@ -8,6 +8,7 @@
 
 #include "doctest_compatibility.h"
 
+#include <cwchar>
 #include <nlohmann/json.hpp>
 using nlohmann::json;
 
@@ -98,15 +99,15 @@ TEST_CASE("wide strings")
             CHECK_THROWS_AS(_ = json::parse(w), json::parse_error&);
 
             // a lone low surrogate cannot start a pair
-            CHECK_THROWS_WITH_AS(_ = json::parse(std::u16string{u'"', 0xDC00, u'"'}), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: ill-formed UTF-8 byte; last read: '\"<U+0000>'", json::parse_error&);
+            CHECK_THROWS_WITH_AS(_ = json::parse(std::u16string{u'"', 0xDC00, u'"'}), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: ill-formed UTF-8 byte; last read: '\"\xFF'", json::parse_error&);
             // a high surrogate followed by a non-low-surrogate unit is invalid
-            CHECK_THROWS_WITH_AS(_ = json::parse(std::u16string{u'"', 0xD800, u'a', u'"'}), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: ill-formed UTF-8 byte; last read: '\"<U+0000>'", json::parse_error&);
+            CHECK_THROWS_WITH_AS(_ = json::parse(std::u16string{u'"', 0xD800, u'a', u'"'}), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: ill-formed UTF-8 byte; last read: '\"\xFF'", json::parse_error&);
             // ... also when the unit is above the low surrogates
-            CHECK_THROWS_WITH_AS(_ = json::parse(std::u16string{u'"', 0xD800, 0xE000, u'"'}), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: ill-formed UTF-8 byte; last read: '\"<U+0000>'", json::parse_error&);
+            CHECK_THROWS_WITH_AS(_ = json::parse(std::u16string{u'"', 0xD800, 0xE000, u'"'}), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: ill-formed UTF-8 byte; last read: '\"\xFF'", json::parse_error&);
             // a lone low surrogate must not swallow the following unit: pairing
             // it with any second unit would produce valid UTF-8, so the error
             // has to report an ill-formed byte at the surrogate's own position
-            CHECK_THROWS_WITH_AS(_ = json::parse(std::u16string{u'"', 0xDC00, u'a', u'"'}), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: ill-formed UTF-8 byte; last read: '\"<U+0000>'", json::parse_error&);
+            CHECK_THROWS_WITH_AS(_ = json::parse(std::u16string{u'"', 0xDC00, u'a', u'"'}), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: ill-formed UTF-8 byte; last read: '\"\xFF'", json::parse_error&);
             // a valid surrogate pair is still decoded (U+1F600)
             CHECK(json::parse(std::u16string{u'"', 0xD83D, 0xDE00, u'"'}).get<std::string>() == "\xF0\x9F\x98\x80");
         }
@@ -141,5 +142,56 @@ TEST_CASE("wide strings")
             CHECK_THROWS_WITH_AS(_ = json::parse(std::u32string{U'"', static_cast<char32_t>(0xFFFFFFFF), U'"'}), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: ill-formed UTF-8 byte; last read: '\"\xFF'", json::parse_error&);
         }
     }
+
+    SECTION("malformed wide-string input outside strings (#5645)")
+    {
+        json _;
+
+        // a lone low surrogate inside a literal must not be truncated to its
+        // low byte and mistaken for the letter the literal expects next
+        // (0xDC72 truncates to 'r', which is what "true" expects after 't')
+        CHECK_THROWS_WITH_AS(_ = json::parse(std::u16string{u't', static_cast<char16_t>(0xDC72), u'u', u'e'}),
+                             "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid literal; last read: 't\xFF'", json::parse_error&);
+
+        // ... also when the lone surrogate is the last unit of the input
+        CHECK_THROWS_WITH_AS(_ = json::parse(std::u16string{u'f', u'a', u'l', u's', static_cast<char16_t>(0xDD65)}),
+                             "[json.exception.parse_error.101] parse error at line 1, column 5: syntax error while parsing value - invalid literal; last read: 'fals\xFF'", json::parse_error&);
+
+        // a high surrogate followed by a unit that is not its low surrogate
+        // must not silently swallow that unit
+        CHECK_THROWS_WITH_AS(_ = json::parse(std::u16string{u't', static_cast<char16_t>(0xD872), u'X', u'u', u'e'}),
+                             "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid literal; last read: 't\xFF'", json::parse_error&);
+
+        // ... in particular, if the swallowed unit is the newline that ends a
+        // // comment, the comment must not extend over the following line
+        CHECK(json::parse(std::u16string{u'[', u'1', u' ', u'/', u'/', static_cast<char16_t>(0xD800), u'\n',
+                                         u',', u'2', u' ', u'/', u'/', u'\n', u']'},
+                          nullptr, true, /*ignore_comments*/true) == json::parse("[1,2]"));
+        CHECK(json::accept(std::u16string{u'[', u'1', u' ', u'/', u'/', static_cast<char16_t>(0xD800), u'\n',
+                                          u',', u'2', u' ', u'/', u'/', u'\n', u']'}, /*ignore_comments*/true));
+
+        // cases 5 and 6 use a 32-bit wchar_t (Linux, macOS, the BSDs) to reach
+        // the UTF-32 helper tested above via u32string; the 16-bit wchar_t of
+        // Windows goes through the UTF-16 helper instead, already covered by
+        // the u16string cases above
+#if WCHAR_MAX > 0xFFFFu
+        // a negative wchar_t must not be mistaken for
+        // char_traits<char>::eof() and silently end the input, letting
+        // trailing garbage pass the strict end-of-input check (only observable
+        // where wint_t is signed, e.g. macOS/the BSDs; on Linux wint_t is
+        // unsigned and this was already handled by #5348)
+        std::wstring w = L"[1]";
+        w.push_back(static_cast<wchar_t>(-1));
+        w += L"garbage";
+        CHECK(!json::accept(w));
+        CHECK_THROWS_WITH_AS(_ = json::parse(w),
+                             "[json.exception.parse_error.101] parse error at line 1, column 4: syntax error while parsing value - invalid literal; last read: '1]\xFF'; expected end of input", json::parse_error&);
+
+        // other negative wchar_t units must not be truncated to their low
+        // byte (0xFFFFFF72 truncates to 'r', as in the u16string case above)
+        CHECK_THROWS_WITH_AS(_ = json::parse(std::wstring{L't', static_cast<wchar_t>(0xFFFFFF72), L'u', L'e'}),
+                             "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid literal; last read: 't\xFF'", json::parse_error&);
+#endif
+    }
 }
 #endif