diff --git a/tests/fuzzing.md b/tests/fuzzing.md index 329cdbf58..8a1a169ea 100644 --- a/tests/fuzzing.md +++ b/tests/fuzzing.md @@ -9,7 +9,16 @@ Each fuzzer driver (`tests/src/fuzzer-parse_*.cpp`) parses its input twice: once once with exceptions. Both calls must agree. Where parsing with exceptions fails, the call without exceptions must return a discarded value (or throw the same kind of non-parse error), and it must never throw a `parse_error`. Where parsing succeeds, both calls must return the same value. The drivers then serialize the value, parse the result back, -and check that nothing was lost. The drivers check all of this with `assert`, so they refuse to build with `NDEBUG`. +and check that nothing was lost. + +Finally, each driver reads the same input through the other input adapters: a `std::istringstream`, a non-contiguous +iterator range (`std::list`), and, on POSIX systems, a `FILE*` from `fmemopen`. The pointer range takes the +fast paths for contiguous input (bulk string scanning, multi-byte loads, lazy position tracking), while the other +adapters read byte by byte. Every adapter must give the same value, or the same exception with the same id, message, +and byte position. The JSON driver checks this for `parse` and `accept`, each with and without `ignore_comments`. The +shared code is `check_input_adapters()` in `tests/src/fuzzer_common.hpp`. + +The drivers check all of this with `assert`, so they refuse to build with `NDEBUG`. ## Corpus creation diff --git a/tests/src/fuzzer-parse_bjdata.cpp b/tests/src/fuzzer-parse_bjdata.cpp index 92e1bed4c..0387e3241 100644 --- a/tests/src/fuzzer-parse_bjdata.cpp +++ b/tests/src/fuzzer-parse_bjdata.cpp @@ -47,12 +47,30 @@ dumps is stable under exactly the same values that break operator==. The unit tests run the same checks on a fixed corpus (see the "BJData round-trip invariants" test case), so keep both in sync. +Finally, it reads the data from a stream, a non-contiguous iterator range, and +a FILE*, which read byte by byte, and checks that each gives the same value or +error (id, message, byte position) as the pointer range, which takes the bulk +fast paths (see check_input_adapters()). + The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer drivers. */ #include "fuzzer_common.hpp" +namespace +{ +// json::from_bjdata for check_input_adapters() +struct read_bjdata +{ + template + json operator()(Args&& ... args) const + { + return json::from_bjdata(std::forward(args)...); + } +}; +} // namespace + // value-stable comparison for the round-trip checks below; see the note // above on why this compares dump()s rather than the json values directly static bool is_value_stable(const json& lhs, const json& rhs) @@ -126,6 +144,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) assert(parsed || noexcept_threw || j_noexcept.is_discarded()); } + // step 3: the other input adapters must read the data alike + check_input_adapters(data, size, read_bjdata()); + // return 0 - non-zero return values are reserved for future use return 0; } diff --git a/tests/src/fuzzer-parse_bon8.cpp b/tests/src/fuzzer-parse_bon8.cpp index 792606d05..bec3e2f93 100644 --- a/tests/src/fuzzer-parse_bon8.cpp +++ b/tests/src/fuzzer-parse_bon8.cpp @@ -17,44 +17,33 @@ array data, it performs the following steps: - j2 = from_bon8(vec) - assert(to_bon8(j2) == vec) -It also checks that reading the data from a stream, which reads strings byte by -byte, gives the same value or error as reading it from contiguous memory, which -copies strings in bulk. +Finally, it reads the data from a stream, a non-contiguous iterator range, and +a FILE*, which read byte by byte, and checks that each gives the same value or +error (id, message, byte position) as the pointer range, which takes the bulk +fast paths (see check_input_adapters()). The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer drivers. */ -#include #include "fuzzer_common.hpp" namespace { -// the serialization of the value read from @a input, or the error message -template -std::string read_bon8(InputType&& input) +// json::from_bon8 for check_input_adapters() +struct read_bon8 { - try + template + json operator()(Args&& ... args) const { - const auto vec = json::to_bon8(json::from_bon8(std::forward(input))); - return {vec.begin(), vec.end()}; + return json::from_bon8(std::forward(args)...); } - catch (const json::exception& e) - { - return e.what(); - } -} +}; } // namespace // see http://llvm.org/docs/LibFuzzer.html extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { - // contiguous and stream input must be read alike - { - std::istringstream stream(std::string(reinterpret_cast(data), size)); - assert(read_bon8(std::vector(data, data + size)) == read_bon8(stream)); - } - std::vector const vec1(data, data + size); // step 0: parse input without exceptions @@ -106,6 +95,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) assert(parsed || noexcept_threw || j_noexcept.is_discarded()); } + // step 3: the other input adapters must read the data alike + check_input_adapters(data, size, read_bon8()); + // return 0 - non-zero return values are reserved for future use return 0; } diff --git a/tests/src/fuzzer-parse_bson.cpp b/tests/src/fuzzer-parse_bson.cpp index 548c30f38..33179de2b 100644 --- a/tests/src/fuzzer-parse_bson.cpp +++ b/tests/src/fuzzer-parse_bson.cpp @@ -17,12 +17,30 @@ array data, it performs the following steps: - j2 = from_bson(vec) - assert(to_bson(j2) == vec) +Finally, it reads the data from a stream, a non-contiguous iterator range, and +a FILE*, which read byte by byte, and checks that each gives the same value or +error (id, message, byte position) as the pointer range, which takes the bulk +fast paths (see check_input_adapters()). + The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer drivers. */ #include "fuzzer_common.hpp" +namespace +{ +// json::from_bson for check_input_adapters() +struct read_bson +{ + template + json operator()(Args&& ... args) const + { + return json::from_bson(std::forward(args)...); + } +}; +} // namespace + // see http://llvm.org/docs/LibFuzzer.html extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { @@ -77,6 +95,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) assert(parsed || noexcept_threw || j_noexcept.is_discarded()); } + // step 3: the other input adapters must read the data alike + check_input_adapters(data, size, read_bson()); + // return 0 - non-zero return values are reserved for future use return 0; } diff --git a/tests/src/fuzzer-parse_cbor.cpp b/tests/src/fuzzer-parse_cbor.cpp index 6d995dc6e..12e7d4104 100644 --- a/tests/src/fuzzer-parse_cbor.cpp +++ b/tests/src/fuzzer-parse_cbor.cpp @@ -17,12 +17,30 @@ array data, it performs the following steps: - j2 = from_cbor(vec) - assert(to_cbor(j2) == vec) +Finally, it reads the data from a stream, a non-contiguous iterator range, and +a FILE*, which read byte by byte, and checks that each gives the same value or +error (id, message, byte position) as the pointer range, which takes the bulk +fast paths (see check_input_adapters()). + The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer drivers. */ #include "fuzzer_common.hpp" +namespace +{ +// json::from_cbor for check_input_adapters() +struct read_cbor +{ + template + json operator()(Args&& ... args) const + { + return json::from_cbor(std::forward(args)...); + } +}; +} // namespace + // see http://llvm.org/docs/LibFuzzer.html extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { @@ -77,6 +95,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) assert(parsed || noexcept_threw || j_noexcept.is_discarded()); } + // step 3: the other input adapters must read the data alike + check_input_adapters(data, size, read_cbor()); + // return 0 - non-zero return values are reserved for future use return 0; } diff --git a/tests/src/fuzzer-parse_json.cpp b/tests/src/fuzzer-parse_json.cpp index 31a811a6c..d656bfb87 100644 --- a/tests/src/fuzzer-parse_json.cpp +++ b/tests/src/fuzzer-parse_json.cpp @@ -17,6 +17,12 @@ array data, it performs the following steps: - j2 = parse(s1) - s2 = serialize(j2) - assert(s1 == s2) +- for ignore_comments = false and true: assert(parse(data) and accept(data) + give the same result for a stream, a non-contiguous iterator range, and a + FILE* as for the pointer range) + +The pointer range takes the bulk fast paths, the other input adapters read byte +by byte; see check_input_adapters(). The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer drivers. @@ -24,6 +30,32 @@ drivers. #include "fuzzer_common.hpp" +namespace +{ +// json::parse and json::accept for check_input_adapters() +struct parse_json +{ + bool ignore_comments; + + template + json operator()(Args&& ... args) const + { + return json::parse(std::forward(args)..., nullptr, true, ignore_comments); + } +}; + +struct accept_json +{ + bool ignore_comments; + + template + bool operator()(Args&& ... args) const + { + return json::accept(std::forward(args)..., ignore_comments); + } +}; +} // namespace + // see http://llvm.org/docs/LibFuzzer.html extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { @@ -76,6 +108,16 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) assert(parsed || noexcept_threw || j_noexcept.is_discarded()); } + // step 3: the other input adapters must read the data alike + for (const bool ignore_comments : + { + false, true + }) + { + check_input_adapters(data, size, parse_json{ignore_comments}); + check_input_adapters(data, size, accept_json{ignore_comments}); + } + // return 0 - non-zero return values are reserved for future use return 0; } diff --git a/tests/src/fuzzer-parse_msgpack.cpp b/tests/src/fuzzer-parse_msgpack.cpp index a5e2e346f..192cbffd8 100644 --- a/tests/src/fuzzer-parse_msgpack.cpp +++ b/tests/src/fuzzer-parse_msgpack.cpp @@ -17,12 +17,30 @@ array data, it performs the following steps: - j2 = from_msgpack(vec) - assert(to_msgpack(j2) == vec) +Finally, it reads the data from a stream, a non-contiguous iterator range, and +a FILE*, which read byte by byte, and checks that each gives the same value or +error (id, message, byte position) as the pointer range, which takes the bulk +fast paths (see check_input_adapters()). + The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer drivers. */ #include "fuzzer_common.hpp" +namespace +{ +// json::from_msgpack for check_input_adapters() +struct read_msgpack +{ + template + json operator()(Args&& ... args) const + { + return json::from_msgpack(std::forward(args)...); + } +}; +} // namespace + // see http://llvm.org/docs/LibFuzzer.html extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { @@ -77,6 +95,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) assert(parsed || noexcept_threw || j_noexcept.is_discarded()); } + // step 3: the other input adapters must read the data alike + check_input_adapters(data, size, read_msgpack()); + // return 0 - non-zero return values are reserved for future use return 0; } diff --git a/tests/src/fuzzer-parse_ubjson.cpp b/tests/src/fuzzer-parse_ubjson.cpp index 641526c90..472857a07 100644 --- a/tests/src/fuzzer-parse_ubjson.cpp +++ b/tests/src/fuzzer-parse_ubjson.cpp @@ -26,12 +26,30 @@ array data, it performs the following steps: The unit tests run the same checks on a fixed corpus (see the "UBJSON round-trip invariants" test case), so keep both in sync. +Finally, it reads the data from a stream, a non-contiguous iterator range, and +a FILE*, which read byte by byte, and checks that each gives the same value or +error (id, message, byte position) as the pointer range, which takes the bulk +fast paths (see check_input_adapters()). + The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer drivers. */ #include "fuzzer_common.hpp" +namespace +{ +// json::from_ubjson for check_input_adapters() +struct read_ubjson +{ + template + json operator()(Args&& ... args) const + { + return json::from_ubjson(std::forward(args)...); + } +}; +} // namespace + // see http://llvm.org/docs/LibFuzzer.html extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { @@ -96,6 +114,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) assert(parsed || noexcept_threw || j_noexcept.is_discarded()); } + // step 3: the other input adapters must read the data alike + check_input_adapters(data, size, read_ubjson()); + // return 0 - non-zero return values are reserved for future use return 0; } diff --git a/tests/src/fuzzer_common.hpp b/tests/src/fuzzer_common.hpp index 4b6c9289c..59b05fdd9 100644 --- a/tests/src/fuzzer_common.hpp +++ b/tests/src/fuzzer_common.hpp @@ -11,6 +11,14 @@ #pragma once #include // assert +#include // size_t +#include // uint8_t +#include // FILE, fmemopen, fclose +#include // list +#include // istringstream +#include // string +#include // forward +#include // vector #include using nlohmann::json; // NOLINT(google-global-names-in-headers): shared by all fuzzer drivers @@ -49,3 +57,89 @@ json parse_without_exceptions(Parse parse, bool& threw) } return {}; } + +// the outcome of reading an input with exceptions: the serialization of the +// value, or the id, message, and (for parse errors) byte of the exception +struct read_outcome +{ + std::string value{}; + int id = 0; + std::string what{}; + std::size_t byte = 0; + + bool operator==(const read_outcome& rhs) const + { + return value == rhs.value && id == rhs.id && what == rhs.what && byte == rhs.byte; + } +}; + +inline std::string outcome_value(const json& j) +{ + return j.dump(-1, ' ', false, json::error_handler_t::keep); +} + +inline std::string outcome_value(bool b) +{ + return b ? "true" : "false"; +} + +template +read_outcome read_with(const Read& read, Args&& ... args) +{ + read_outcome result; + try + { + result.value = outcome_value(read(std::forward(args)...)); + } + catch (const json::parse_error& e) + { + result.id = e.id; + result.what = e.what(); + result.byte = e.byte; + } + catch (const json::exception& e) + { + result.id = e.id; + result.what = e.what(); + } + return result; +} + +// The contiguous input (a pointer range) takes the bulk fast paths; the other +// input adapters read byte by byte. Reads @a data through each of them and +// checks that all give the same value, or the same exception with the same +// id, message, and byte position. +// +// @a read is called with either an iterator range or a single input (stream +// or FILE*) and returns the parsed value (or the result of accept()). +template +void check_input_adapters(const std::uint8_t* data, std::size_t size, const Read& read) +{ + const read_outcome expected = read_with(read, data, data + size); + + // stream adapter + { + std::istringstream stream(std::string(reinterpret_cast(data), size)); + assert(read_with(read, stream) == expected); + } + + // iterator adapter on a non-contiguous range + { + const std::list list(data, data + size); + assert(read_with(read, list.begin(), list.end()) == expected); + } + +#if defined(__unix__) || defined(__APPLE__) + // FILE* adapter; fmemopen may refuse an empty buffer + { + std::vector buffer(data, data + size); + std::FILE* file = buffer.empty() ? nullptr : fmemopen(buffer.data(), buffer.size(), "rb"); + if (file != nullptr) + { + const read_outcome actual = read_with(read, file); + std::fclose(file); + assert(actual == expected); + } + } +#endif +}