Keep a NUL byte ending a // comment as the end of input

With the default NUL handling (JSON_STRICT_NUL_HANDLING not set), a NUL
byte in the input is treated as the real end of input everywhere -
except when it immediately ends a `//` comment: scan_comment() matched
'\0' as a comment terminator like '\n', so the NUL was consumed as
part of the comment and scan() never saw it as end of input; the next
get() then kept reading past it. Multi-line comments and
JSON_STRICT_NUL_HANDLING=1 were unaffected, since there the NUL is
just part of the comment text.

Fix scan_comment() to leave the NUL unconsumed (unget()) instead of
returning it as part of the comment, so the following scan() reports
it as end of input, exactly as for a NUL anywhere else.

Fixes #5659.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

docs/mkdocs/docs/features/types/template_parameters.md

@@ -389,7 +389,6 @@ using array_t = ArrayType<basic_json, AllocatorType<basic_json>>;
 | Functionality                                                                                                                     | Additional requirement                                                                                                                                                                       |
 |-----------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | [`diff`](../../api/basic_json/diff.md), [`items`](../../api/basic_json/items.md), [`std::hash`](../../api/basic_json/std_hash.md) | conversion of a `#!cpp std::size_t` to `StringType`: either assignability from the result of `#!cpp std::to_string`, or an ADL overload `#!cpp void int_to_string(StringType&, std::size_t)` |
-| [`operator/(std::size_t)`](../../api/json_pointer/operator_slash.md)                                                              | the same conversion of a `#!cpp std::size_t` to `StringType` as `diff`, `items`, and `std::hash` above                                                                                       |
 | [`std::hash<basic_json>`](../../api/basic_json/std_hash.md)                                                                       | additionally a specialization of `#!cpp std::hash<StringType>`                                                                                                                               |
 | [`to_bson`](../../api/basic_json/to_bson.md)                                                                                      | `find(value_type)` and `npos`                                                                                                                                                                |
 | [`parse`](../../api/basic_json/parse.md) from a `string_t`                                                                        | the input adapters must accept it; otherwise pass a character range                                                                                                                          |

include/nlohmann/detail/input/lexer.hpp

@@ -975,10 +975,15 @@ class lexer : public lexer_base<BasicJsonType>
                         case '\n':
                         case '\r':
                         case char_traits<char_type>::eof():
+                            return true;
+
 #if !JSON_STRICT_NUL_HANDLING
                         case '\0':
-#endif
+                            // a NUL byte is the end of the input (see scan()),
+                            // so leave it for scan() to see
+                            unget();
                             return true;
+#endif
 
                         default:
                             break;

include/nlohmann/detail/json_pointer.hpp

@@ -26,7 +26,6 @@
 #include <nlohmann/detail/macro_scope.hpp>
 #include <nlohmann/detail/string_concat.hpp>
 #include <nlohmann/detail/string_escape.hpp>
-#include <nlohmann/detail/string_utils.hpp>
 #include <nlohmann/detail/value_t.hpp>
 
 NLOHMANN_JSON_NAMESPACE_BEGIN
@@ -117,7 +116,7 @@ class json_pointer
     /// @sa https://json.nlohmann.me/api/json_pointer/operator_slasheq/
     json_pointer& operator/=(std::size_t array_idx)
     {
-        return *this /= detail::to_string<string_t>(array_idx);
+        return *this /= std::to_string(array_idx);
     }
 
     /// @brief create a new JSON pointer by appending the right JSON pointer at the end of the left JSON pointer
@@ -747,7 +746,7 @@ class json_pointer
                         // "-" always fails the range check
                         return false;
                     }
-                    if (JSON_HEDLEY_UNLIKELY(reference_token.size() == 1 && !('0' <= reference_token[0] && reference_token[0] <= '9')))
+                    if (JSON_HEDLEY_UNLIKELY(reference_token.size() == 1 && !("0" <= reference_token && reference_token <= "9")))
                     {
                         // invalid char
                         return false;
@@ -775,7 +774,7 @@ class json_pointer
                     // not throw (see #5395), so such a reference token is treated as "not found"
                     errno = 0; // strtoull() does not reset errno on success
                     char* p_end = nullptr; // NOLINT(misc-const-correctness)
-                    const unsigned long long magnitude = std::strtoull(reference_token.data(), &p_end, 10); // NOLINT(runtime/int)
+                    const unsigned long long magnitude = std::strtoull(reference_token.c_str(), &p_end, 10); // NOLINT(runtime/int)
                     if (JSON_HEDLEY_UNLIKELY(errno == ERANGE // the value exceeds ULLONG_MAX
                                              || magnitude >= static_cast<unsigned long long>((std::numeric_limits<typename BasicJsonType::size_type>::max)()))) // NOLINT(runtime/int)
                     {

single_include/nlohmann/json.hpp

@@ -10067,10 +10067,15 @@ class lexer : public lexer_base<BasicJsonType>
                         case '\n':
                         case '\r':
                         case char_traits<char_type>::eof():
+                            return true;
+
 #if !JSON_STRICT_NUL_HANDLING
                         case '\0':
-#endif
+                            // a NUL byte is the end of the input (see scan()),
+                            // so leave it for scan() to see
+                            unget();
                             return true;
+#endif
 
                         default:
                             break;
@@ -18867,8 +18872,6 @@ NLOHMANN_JSON_NAMESPACE_END
 
 // #include <nlohmann/detail/string_escape.hpp>
 
-// #include <nlohmann/detail/string_utils.hpp>
-
 // #include <nlohmann/detail/value_t.hpp>
 
 
@@ -18960,7 +18963,7 @@ class json_pointer
     /// @sa https://json.nlohmann.me/api/json_pointer/operator_slasheq/
     json_pointer& operator/=(std::size_t array_idx)
     {
-        return *this /= detail::to_string<string_t>(array_idx);
+        return *this /= std::to_string(array_idx);
     }
 
     /// @brief create a new JSON pointer by appending the right JSON pointer at the end of the left JSON pointer
@@ -19590,7 +19593,7 @@ class json_pointer
                         // "-" always fails the range check
                         return false;
                     }
-                    if (JSON_HEDLEY_UNLIKELY(reference_token.size() == 1 && !('0' <= reference_token[0] && reference_token[0] <= '9')))
+                    if (JSON_HEDLEY_UNLIKELY(reference_token.size() == 1 && !("0" <= reference_token && reference_token <= "9")))
                     {
                         // invalid char
                         return false;
@@ -19618,7 +19621,7 @@ class json_pointer
                     // not throw (see #5395), so such a reference token is treated as "not found"
                     errno = 0; // strtoull() does not reset errno on success
                     char* p_end = nullptr; // NOLINT(misc-const-correctness)
-                    const unsigned long long magnitude = std::strtoull(reference_token.data(), &p_end, 10); // NOLINT(runtime/int)
+                    const unsigned long long magnitude = std::strtoull(reference_token.c_str(), &p_end, 10); // NOLINT(runtime/int)
                     if (JSON_HEDLEY_UNLIKELY(errno == ERANGE // the value exceeds ULLONG_MAX
                                              || magnitude >= static_cast<unsigned long long>((std::numeric_limits<typename BasicJsonType::size_type>::max)()))) // NOLINT(runtime/int)
                     {

tests/src/unit-alt-string.cpp

@@ -343,41 +343,6 @@ TEST_CASE("alternative string type")
         CHECK(j2.flatten().unflatten() == j2);
     }
 
-    SECTION("contains(json_pointer)")
-    {
-        // contains(json_pointer) must compile and work with a string_t that has
-        // no c_str() and no comparison with const char* (see #5666)
-        auto j = alt_json::parse(R"({"foo": ["bar", "baz"]})");
-
-        // present: object key and array indices
-        CHECK(j.contains(alt_json::json_pointer("/foo")));
-        CHECK(j.contains(alt_json::json_pointer("/foo/0")));
-        CHECK(j.contains(alt_json::json_pointer("/foo/1")));
-
-        // missing: absent object key and out-of-range array index
-        CHECK_FALSE(j.contains(alt_json::json_pointer("/bar")));
-        CHECK_FALSE(j.contains(alt_json::json_pointer("/foo/2")));
-
-        // "-" always fails the range check
-        CHECK_FALSE(j.contains(alt_json::json_pointer("/foo/-")));
-
-        // an array index must not have a leading zero
-        CHECK_FALSE(j.contains(alt_json::json_pointer("/foo/01")));
-
-        // a reference token that is not a number
-        CHECK_FALSE(j.contains(alt_json::json_pointer("/foo/bar")));
-    }
-
-    SECTION("operator/(std::size_t)")
-    {
-        // json_pointer::operator/=(std::size_t) must compile without string_t
-        // being constructible from std::string (see #5666)
-        auto j = alt_json::parse(R"({"foo": ["bar", "baz"]})");
-
-        CHECK(j.at(alt_json::json_pointer("/foo") / std::size_t(0)) == j["foo"][0]);
-        CHECK(j.at(alt_json::json_pointer("/foo") / std::size_t(1)) == j["foo"][1]);
-    }
-
     SECTION("patch")
     {
         alt_json const patch1 = alt_json::parse(R"([{ "op": "add", "path": "/a/b", "value": [ "foo", "bar" ] }])");

tests/src/unit-class_parser.cpp

@@ -592,6 +592,45 @@ TEST_CASE("parser class")
 
                 // parsing from a string literal is unaffected either way
                 CHECK(json::parse("123") == json(123));
+
+                // a NUL byte that ends a // comment ends the input just
+                // like a NUL byte anywhere else (issue #5659); before the
+                // fix, the NUL was consumed as part of the comment, and
+                // scanning continued with whatever followed it
+                {
+                    // same as "//c" alone (real end of input after the
+                    // comment), rather than continuing with "[1]"
+                    std::string s1 = "//c";
+                    s1.push_back('\0');
+                    s1 += "[1]";
+                    json _; // NOLINT(readability-identifier-naming)
+                    CHECK_THROWS_WITH_AS(_ = json::parse(s1, nullptr, true, true),
+                                         "[json.exception.parse_error.101] parse error at line 1, column 4: syntax error while parsing value - unexpected end of input; expected '[', '{', or a literal",
+                                         json::parse_error&);
+                    CHECK_FALSE(json::accept(s1, true, true));
+                }
+
+                {
+                    // same as "[1, //c" alone, rather than continuing with " 2]"
+                    std::string s2 = "[1, //c";
+                    s2.push_back('\0');
+                    s2 += " 2]";
+                    json _; // NOLINT(readability-identifier-naming)
+                    CHECK_THROWS_WITH_AS(_ = json::parse(s2, nullptr, true, true),
+                                         "[json.exception.parse_error.101] parse error at line 1, column 8: syntax error while parsing value - unexpected end of input; expected '[', '{', or a literal",
+                                         json::parse_error&);
+                    CHECK_FALSE(json::accept(s2, true, true));
+                }
+
+                {
+                    // same as "1 //c" alone: the comment (and the NUL that
+                    // ends it) is ignored, and "x" is never reached
+                    std::string s3 = "1 //c";
+                    s3.push_back('\0');
+                    s3 += "x";
+                    CHECK(json::parse(s3, nullptr, true, true) == json(1));
+                    CHECK(json::accept(s3, true, true));
+                }
             }
 #endif