Keep a NUL byte ending a // comment as the end of input

With the default NUL handling (JSON_STRICT_NUL_HANDLING not set), a NUL
byte in the input is treated as the real end of input everywhere -
except when it immediately ends a `//` comment: scan_comment() matched
'\0' as a comment terminator like '\n', so the NUL was consumed as
part of the comment and scan() never saw it as end of input; the next
get() then kept reading past it. Multi-line comments and
JSON_STRICT_NUL_HANDLING=1 were unaffected, since there the NUL is
just part of the comment text.

Fix scan_comment() to leave the NUL unconsumed (unget()) instead of
returning it as part of the comment, so the following scan() reports
it as end of input, exactly as for a NUL anywhere else.

Fixes #5659.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

include/nlohmann/detail/input/lexer.hpp

@@ -975,10 +975,15 @@ class lexer : public lexer_base<BasicJsonType>
                         case '\n':
                         case '\r':
                         case char_traits<char_type>::eof():
+                            return true;
+
 #if !JSON_STRICT_NUL_HANDLING
                         case '\0':
-#endif
+                            // a NUL byte is the end of the input (see scan()),
+                            // so leave it for scan() to see
+                            unget();
                             return true;
+#endif
 
                         default:
                             break;

single_include/nlohmann/json.hpp

@@ -10067,10 +10067,15 @@ class lexer : public lexer_base<BasicJsonType>
                         case '\n':
                         case '\r':
                         case char_traits<char_type>::eof():
+                            return true;
+
 #if !JSON_STRICT_NUL_HANDLING
                         case '\0':
-#endif
+                            // a NUL byte is the end of the input (see scan()),
+                            // so leave it for scan() to see
+                            unget();
                             return true;
+#endif
 
                         default:
                             break;

tests/src/unit-class_parser.cpp

@@ -592,6 +592,45 @@ TEST_CASE("parser class")
 
                 // parsing from a string literal is unaffected either way
                 CHECK(json::parse("123") == json(123));
+
+                // a NUL byte that ends a // comment ends the input just
+                // like a NUL byte anywhere else (issue #5659); before the
+                // fix, the NUL was consumed as part of the comment, and
+                // scanning continued with whatever followed it
+                {
+                    // same as "//c" alone (real end of input after the
+                    // comment), rather than continuing with "[1]"
+                    std::string s1 = "//c";
+                    s1.push_back('\0');
+                    s1 += "[1]";
+                    json _; // NOLINT(readability-identifier-naming)
+                    CHECK_THROWS_WITH_AS(_ = json::parse(s1, nullptr, true, true),
+                                         "[json.exception.parse_error.101] parse error at line 1, column 4: syntax error while parsing value - unexpected end of input; expected '[', '{', or a literal",
+                                         json::parse_error&);
+                    CHECK_FALSE(json::accept(s1, true, true));
+                }
+
+                {
+                    // same as "[1, //c" alone, rather than continuing with " 2]"
+                    std::string s2 = "[1, //c";
+                    s2.push_back('\0');
+                    s2 += " 2]";
+                    json _; // NOLINT(readability-identifier-naming)
+                    CHECK_THROWS_WITH_AS(_ = json::parse(s2, nullptr, true, true),
+                                         "[json.exception.parse_error.101] parse error at line 1, column 8: syntax error while parsing value - unexpected end of input; expected '[', '{', or a literal",
+                                         json::parse_error&);
+                    CHECK_FALSE(json::accept(s2, true, true));
+                }
+
+                {
+                    // same as "1 //c" alone: the comment (and the NUL that
+                    // ends it) is ignored, and "x" is never reached
+                    std::string s3 = "1 //c";
+                    s3.push_back('\0');
+                    s3 += "x";
+                    CHECK(json::parse(s3, nullptr, true, true) == json(1));
+                    CHECK(json::accept(s3, true, true));
+                }
             }
 #endif
 

tests/src/unit-regression2.cpp

@@ -766,25 +766,6 @@ TEST_CASE("regression tests 2")
         CHECK(j == k);
     }
 
-#ifdef JSON_HAS_CPP_17
-    SECTION("issue #5066 - MSVC converts json to std::variant<json> via the conversion operator")
-    {
-        // std::variant<json> must not be retrievable via get<>(), because otherwise the
-        // implicit conversion operator becomes a candidate that MSVC picks over the variant's
-        // converting constructor, routing a number through the string from_json overload
-        static_assert(!nlohmann::detail::is_detected<nlohmann::detail::get_template_function, const json&, std::variant<json>>::value,
-                      "std::variant<json> must not be retrievable via get<>()");
-
-        // clang before 7 cannot instantiate libstdc++'s std::variant<json>
-#if !(defined(__clang__) && __clang_major__ < 7)
-        // push_back, not emplace_back: #5066 needs the implicit conversion
-        // from json to the vector's value type
-        std::vector<std::variant<json>> v;
-        v.push_back(json(1)); // NOLINT(hicpp-use-emplace,modernize-use-emplace)
-        CHECK(std::get<0>(v[0]) == 1);
-#endif
-    }
-#endif
 }
 
 TEST_CASE("regression test - parser callback must not lose a duplicate key's prior value")