Fuzz the streaming input adapters against the contiguous one

The fast paths for contiguous input (bulk string scanning, multi-byte
loads, lazy position tracking) are taken only when parsing from a
pointer range; streams and non-contiguous iterators still read byte by
byte. The fuzzer drivers parsed only contiguous input, so a difference
between the two would ship unnoticed.

check_input_adapters() in fuzzer_common.hpp reads the input through a
std::istringstream, a std::list iterator range, and (on POSIX) a FILE*
from fmemopen, and asserts the same value, or the same exception id,
message and byte position, as for the pointer range. All seven drivers
call it; the JSON driver checks parse and accept, each with and without
ignore_comments. It replaces the stream check of the BON8 driver.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

tests/fuzzing.md

@@ -9,7 +9,16 @@ Each fuzzer driver (`tests/src/fuzzer-parse_*.cpp`) parses its input twice: once
 once with exceptions. Both calls must agree. Where parsing with exceptions fails, the call without exceptions must
 return a discarded value (or throw the same kind of non-parse error), and it must never throw a `parse_error`. Where
 parsing succeeds, both calls must return the same value. The drivers then serialize the value, parse the result back,
-and check that nothing was lost. The drivers check all of this with `assert`, so they refuse to build with `NDEBUG`.
+and check that nothing was lost.
+
+Finally, each driver reads the same input through the other input adapters: a `std::istringstream`, a non-contiguous
+iterator range (`std::list<std::uint8_t>`), and, on POSIX systems, a `FILE*` from `fmemopen`. The pointer range takes the
+fast paths for contiguous input (bulk string scanning, multi-byte loads, lazy position tracking), while the other
+adapters read byte by byte. Every adapter must give the same value, or the same exception with the same id, message,
+and byte position. The JSON driver checks this for `parse` and `accept`, each with and without `ignore_comments`. The
+shared code is `check_input_adapters()` in `tests/src/fuzzer_common.hpp`.
+
+The drivers check all of this with `assert`, so they refuse to build with `NDEBUG`.
 
 ## Corpus creation
 

tests/src/fuzzer-parse_bjdata.cpp

@@ -47,12 +47,30 @@ dumps is stable under exactly the same values that break operator==.
 The unit tests run the same checks on a fixed corpus (see the "BJData round-trip
 invariants" test case), so keep both in sync.
 
+Finally, it reads the data from a stream, a non-contiguous iterator range, and
+a FILE*, which read byte by byte, and checks that each gives the same value or
+error (id, message, byte position) as the pointer range, which takes the bulk
+fast paths (see check_input_adapters()).
+
 The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer
 drivers.
 */
 
 #include "fuzzer_common.hpp"
 
+namespace
+{
+// json::from_bjdata for check_input_adapters()
+struct read_bjdata
+{
+    template<typename... Args>
+    json operator()(Args&& ... args) const
+    {
+        return json::from_bjdata(std::forward<Args>(args)...);
+    }
+};
+} // namespace
+
 // value-stable comparison for the round-trip checks below; see the note
 // above on why this compares dump()s rather than the json values directly
 static bool is_value_stable(const json& lhs, const json& rhs)
@@ -126,6 +144,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
         assert(parsed || noexcept_threw || j_noexcept.is_discarded());
     }
 
+    // step 3: the other input adapters must read the data alike
+    check_input_adapters(data, size, read_bjdata());
+
     // return 0 - non-zero return values are reserved for future use
     return 0;
 }

tests/src/fuzzer-parse_bon8.cpp

@@ -17,44 +17,33 @@ array data, it performs the following steps:
 - j2 = from_bon8(vec)
 - assert(to_bon8(j2) == vec)
 
-It also checks that reading the data from a stream, which reads strings byte by
-byte, gives the same value or error as reading it from contiguous memory, which
-copies strings in bulk.
+Finally, it reads the data from a stream, a non-contiguous iterator range, and
+a FILE*, which read byte by byte, and checks that each gives the same value or
+error (id, message, byte position) as the pointer range, which takes the bulk
+fast paths (see check_input_adapters()).
 
 The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer
 drivers.
 */
 
-#include <sstream>
 #include "fuzzer_common.hpp"
 
 namespace
 {
-// the serialization of the value read from @a input, or the error message
-template<typename InputType>
-std::string read_bon8(InputType&& input)
+// json::from_bon8 for check_input_adapters()
+struct read_bon8
 {
-    try
+    template<typename... Args>
+    json operator()(Args&& ... args) const
     {
-        const auto vec = json::to_bon8(json::from_bon8(std::forward<InputType>(input)));
-        return {vec.begin(), vec.end()};
+        return json::from_bon8(std::forward<Args>(args)...);
     }
-    catch (const json::exception& e)
-    {
-        return e.what();
-    }
-}
+};
 } // namespace
 
 // see http://llvm.org/docs/LibFuzzer.html
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
 {
-    // contiguous and stream input must be read alike
-    {
-        std::istringstream stream(std::string(reinterpret_cast<const char*>(data), size));
-        assert(read_bon8(std::vector<uint8_t>(data, data + size)) == read_bon8(stream));
-    }
-
     std::vector<uint8_t> const vec1(data, data + size);
 
     // step 0: parse input without exceptions
@@ -106,6 +95,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
         assert(parsed || noexcept_threw || j_noexcept.is_discarded());
     }
 
+    // step 3: the other input adapters must read the data alike
+    check_input_adapters(data, size, read_bon8());
+
     // return 0 - non-zero return values are reserved for future use
     return 0;
 }

tests/src/fuzzer-parse_bson.cpp

@@ -17,12 +17,30 @@ array data, it performs the following steps:
 - j2 = from_bson(vec)
 - assert(to_bson(j2) == vec)
 
+Finally, it reads the data from a stream, a non-contiguous iterator range, and
+a FILE*, which read byte by byte, and checks that each gives the same value or
+error (id, message, byte position) as the pointer range, which takes the bulk
+fast paths (see check_input_adapters()).
+
 The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer
 drivers.
 */
 
 #include "fuzzer_common.hpp"
 
+namespace
+{
+// json::from_bson for check_input_adapters()
+struct read_bson
+{
+    template<typename... Args>
+    json operator()(Args&& ... args) const
+    {
+        return json::from_bson(std::forward<Args>(args)...);
+    }
+};
+} // namespace
+
 // see http://llvm.org/docs/LibFuzzer.html
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
 {
@@ -77,6 +95,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
         assert(parsed || noexcept_threw || j_noexcept.is_discarded());
     }
 
+    // step 3: the other input adapters must read the data alike
+    check_input_adapters(data, size, read_bson());
+
     // return 0 - non-zero return values are reserved for future use
     return 0;
 }

tests/src/fuzzer-parse_cbor.cpp

@@ -17,12 +17,30 @@ array data, it performs the following steps:
 - j2 = from_cbor(vec)
 - assert(to_cbor(j2) == vec)
 
+Finally, it reads the data from a stream, a non-contiguous iterator range, and
+a FILE*, which read byte by byte, and checks that each gives the same value or
+error (id, message, byte position) as the pointer range, which takes the bulk
+fast paths (see check_input_adapters()).
+
 The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer
 drivers.
 */
 
 #include "fuzzer_common.hpp"
 
+namespace
+{
+// json::from_cbor for check_input_adapters()
+struct read_cbor
+{
+    template<typename... Args>
+    json operator()(Args&& ... args) const
+    {
+        return json::from_cbor(std::forward<Args>(args)...);
+    }
+};
+} // namespace
+
 // see http://llvm.org/docs/LibFuzzer.html
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
 {
@@ -77,6 +95,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
         assert(parsed || noexcept_threw || j_noexcept.is_discarded());
     }
 
+    // step 3: the other input adapters must read the data alike
+    check_input_adapters(data, size, read_cbor());
+
     // return 0 - non-zero return values are reserved for future use
     return 0;
 }

tests/src/fuzzer-parse_json.cpp

@@ -17,6 +17,12 @@ array data, it performs the following steps:
 - j2 = parse(s1)
 - s2 = serialize(j2)
 - assert(s1 == s2)
+- for ignore_comments = false and true: assert(parse(data) and accept(data)
+  give the same result for a stream, a non-contiguous iterator range, and a
+  FILE* as for the pointer range)
+
+The pointer range takes the bulk fast paths, the other input adapters read byte
+by byte; see check_input_adapters().
 
 The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer
 drivers.
@@ -24,6 +30,32 @@ drivers.
 
 #include "fuzzer_common.hpp"
 
+namespace
+{
+// json::parse and json::accept for check_input_adapters()
+struct parse_json
+{
+    bool ignore_comments;
+
+    template<typename... Args>
+    json operator()(Args&& ... args) const
+    {
+        return json::parse(std::forward<Args>(args)..., nullptr, true, ignore_comments);
+    }
+};
+
+struct accept_json
+{
+    bool ignore_comments;
+
+    template<typename... Args>
+    bool operator()(Args&& ... args) const
+    {
+        return json::accept(std::forward<Args>(args)..., ignore_comments);
+    }
+};
+} // namespace
+
 // see http://llvm.org/docs/LibFuzzer.html
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
 {
@@ -76,6 +108,16 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
         assert(parsed || noexcept_threw || j_noexcept.is_discarded());
     }
 
+    // step 3: the other input adapters must read the data alike
+    for (const bool ignore_comments :
+            {
+                false, true
+            })
+    {
+        check_input_adapters(data, size, parse_json{ignore_comments});
+        check_input_adapters(data, size, accept_json{ignore_comments});
+    }
+
     // return 0 - non-zero return values are reserved for future use
     return 0;
 }

tests/src/fuzzer-parse_msgpack.cpp

@@ -17,12 +17,30 @@ array data, it performs the following steps:
 - j2 = from_msgpack(vec)
 - assert(to_msgpack(j2) == vec)
 
+Finally, it reads the data from a stream, a non-contiguous iterator range, and
+a FILE*, which read byte by byte, and checks that each gives the same value or
+error (id, message, byte position) as the pointer range, which takes the bulk
+fast paths (see check_input_adapters()).
+
 The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer
 drivers.
 */
 
 #include "fuzzer_common.hpp"
 
+namespace
+{
+// json::from_msgpack for check_input_adapters()
+struct read_msgpack
+{
+    template<typename... Args>
+    json operator()(Args&& ... args) const
+    {
+        return json::from_msgpack(std::forward<Args>(args)...);
+    }
+};
+} // namespace
+
 // see http://llvm.org/docs/LibFuzzer.html
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
 {
@@ -77,6 +95,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
         assert(parsed || noexcept_threw || j_noexcept.is_discarded());
     }
 
+    // step 3: the other input adapters must read the data alike
+    check_input_adapters(data, size, read_msgpack());
+
     // return 0 - non-zero return values are reserved for future use
     return 0;
 }

tests/src/fuzzer-parse_ubjson.cpp

@@ -26,12 +26,30 @@ array data, it performs the following steps:
 The unit tests run the same checks on a fixed corpus (see the "UBJSON round-trip
 invariants" test case), so keep both in sync.
 
+Finally, it reads the data from a stream, a non-contiguous iterator range, and
+a FILE*, which read byte by byte, and checks that each gives the same value or
+error (id, message, byte position) as the pointer range, which takes the bulk
+fast paths (see check_input_adapters()).
+
 The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer
 drivers.
 */
 
 #include "fuzzer_common.hpp"
 
+namespace
+{
+// json::from_ubjson for check_input_adapters()
+struct read_ubjson
+{
+    template<typename... Args>
+    json operator()(Args&& ... args) const
+    {
+        return json::from_ubjson(std::forward<Args>(args)...);
+    }
+};
+} // namespace
+
 // see http://llvm.org/docs/LibFuzzer.html
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
 {
@@ -96,6 +114,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
         assert(parsed || noexcept_threw || j_noexcept.is_discarded());
     }
 
+    // step 3: the other input adapters must read the data alike
+    check_input_adapters(data, size, read_ubjson());
+
     // return 0 - non-zero return values are reserved for future use
     return 0;
 }

tests/src/fuzzer_common.hpp

@@ -11,6 +11,14 @@
 #pragma once
 
 #include <cassert> // assert
+#include <cstddef> // size_t
+#include <cstdint> // uint8_t
+#include <cstdio> // FILE, fmemopen, fclose
+#include <list> // list
+#include <sstream> // istringstream
+#include <string> // string
+#include <utility> // forward
+#include <vector> // vector
 #include <nlohmann/json.hpp>
 
 using nlohmann::json; // NOLINT(google-global-names-in-headers): shared by all fuzzer drivers
@@ -49,3 +57,89 @@ json parse_without_exceptions(Parse parse, bool& threw)
     }
     return {};
 }
+
+// the outcome of reading an input with exceptions: the serialization of the
+// value, or the id, message, and (for parse errors) byte of the exception
+struct read_outcome
+{
+    std::string value{};
+    int id = 0;
+    std::string what{};
+    std::size_t byte = 0;
+
+    bool operator==(const read_outcome& rhs) const
+    {
+        return value == rhs.value && id == rhs.id && what == rhs.what && byte == rhs.byte;
+    }
+};
+
+inline std::string outcome_value(const json& j)
+{
+    return j.dump(-1, ' ', false, json::error_handler_t::keep);
+}
+
+inline std::string outcome_value(bool b)
+{
+    return b ? "true" : "false";
+}
+
+template<typename Read, typename... Args>
+read_outcome read_with(const Read& read, Args&& ... args)
+{
+    read_outcome result;
+    try
+    {
+        result.value = outcome_value(read(std::forward<Args>(args)...));
+    }
+    catch (const json::parse_error& e)
+    {
+        result.id = e.id;
+        result.what = e.what();
+        result.byte = e.byte;
+    }
+    catch (const json::exception& e)
+    {
+        result.id = e.id;
+        result.what = e.what();
+    }
+    return result;
+}
+
+// The contiguous input (a pointer range) takes the bulk fast paths; the other
+// input adapters read byte by byte. Reads @a data through each of them and
+// checks that all give the same value, or the same exception with the same
+// id, message, and byte position.
+//
+// @a read is called with either an iterator range or a single input (stream
+// or FILE*) and returns the parsed value (or the result of accept()).
+template<typename Read>
+void check_input_adapters(const std::uint8_t* data, std::size_t size, const Read& read)
+{
+    const read_outcome expected = read_with(read, data, data + size);
+
+    // stream adapter
+    {
+        std::istringstream stream(std::string(reinterpret_cast<const char*>(data), size));
+        assert(read_with(read, stream) == expected);
+    }
+
+    // iterator adapter on a non-contiguous range
+    {
+        const std::list<std::uint8_t> list(data, data + size);
+        assert(read_with(read, list.begin(), list.end()) == expected);
+    }
+
+#if defined(__unix__) || defined(__APPLE__)
+    // FILE* adapter; fmemopen may refuse an empty buffer
+    {
+        std::vector<std::uint8_t> buffer(data, data + size);
+        std::FILE* file = buffer.empty() ? nullptr : fmemopen(buffer.data(), buffer.size(), "rb");
+        if (file != nullptr)
+        {
+            const read_outcome actual = read_with(read, file);
+            std::fclose(file);
+            assert(actual == expected);
+        }
+    }
+#endif
+}