From 467670aa7a371be9676f28d2e1148c375d528a34 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Wed, 14 Oct 2026 16:24:28 +0200 Subject: [PATCH] Test that the contiguous fast paths never read outside the input The bulk string scanner, the SWAR searches, the UTF-8 prefix validator, the 8-digit number loop, the 8-byte indentation skip, and the BSON/BON8 bulk string readers load several bytes at a time from contiguous input. ASan misses over-reads into the slack of the same heap allocation, and fuzzers rarely end an input exactly at a page boundary. The new test copies each input directly in front of, and directly behind, an inaccessible page (mmap/mprotect on POSIX, VirtualAlloc/VirtualProtect on Windows), so reading even one byte outside the input crashes. It covers the string-scan functions directly and json::parse, accept, sax_parse (with and without comments) and all binary readers through the public API, with inputs ending at every byte of strings, escapes, UTF-8 sequences, numbers, literals, indentation runs, and binary documents. Each result must match parsing the same input from a std::string. Signed-off-by: Niels Lohmann --- tests/src/unit-guard-page.cpp | 422 ++++++++++++++++++++++++++++++++++ 1 file changed, 422 insertions(+) create mode 100644 tests/src/unit-guard-page.cpp diff --git a/tests/src/unit-guard-page.cpp b/tests/src/unit-guard-page.cpp new file mode 100644 index 000000000..d8ea4fba3 --- /dev/null +++ b/tests/src/unit-guard-page.cpp @@ -0,0 +1,422 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +// The fast paths for contiguous input (bulk string scanning, SWAR searches, +// UTF-8 prefix validation, the 8-digit number loop, the 8-byte indentation +// skip, the BSON/BON8 bulk string readers) load several bytes at a time. This +// test places every input directly in front of (or directly behind) an +// inaccessible page, so reading even one byte outside the input crashes, with +// or without sanitizers. ASan alone misses over-reads into the slack of the +// same heap allocation. + +#include "doctest_compatibility.h" + +#include +using nlohmann::json; + +#include +#include +#include +#include +#include + +#if defined(_WIN32) + #define JSON_TEST_GUARD_PAGE_WINDOWS + #ifndef NOMINMAX + #define NOMINMAX + #endif + #ifndef WIN32_LEAN_AND_MEAN + #define WIN32_LEAN_AND_MEAN + #endif + #include +#elif defined(__unix__) || defined(__APPLE__) + #define JSON_TEST_GUARD_PAGE_POSIX + #include + #include +#endif + +#if defined(JSON_TEST_GUARD_PAGE_WINDOWS) || defined(JSON_TEST_GUARD_PAGE_POSIX) + +namespace +{ +// Three pages: an inaccessible one, a readable one, an inaccessible one. An +// input is copied either to the end of the middle page (any read past its last +// byte faults) or to its start (any read before its first byte faults). +class guarded_buffer +{ + public: + guarded_buffer() + : page(page_size()) + { +#if defined(JSON_TEST_GUARD_PAGE_WINDOWS) + void* const p = VirtualAlloc(nullptr, 3 * page, MEM_RESERVE | MEM_COMMIT, PAGE_NOACCESS); + REQUIRE(p != nullptr); + base = static_cast(p); + DWORD old_protect = 0; + REQUIRE(VirtualProtect(base + page, page, PAGE_READWRITE, &old_protect) != 0); +#else + void* const p = mmap(nullptr, 3 * page, PROT_NONE, MAP_PRIVATE | MAP_ANON, -1, 0); + REQUIRE(p != MAP_FAILED); + base = static_cast(p); + REQUIRE(mprotect(base + page, page, PROT_READ | PROT_WRITE) == 0); +#endif + } + + ~guarded_buffer() + { +#if defined(JSON_TEST_GUARD_PAGE_WINDOWS) + VirtualFree(base, 0, MEM_RELEASE); +#else + munmap(base, 3 * page); +#endif + } + + guarded_buffer(const guarded_buffer&) = delete; + guarded_buffer& operator=(const guarded_buffer&) = delete; + guarded_buffer(guarded_buffer&&) = delete; + guarded_buffer& operator=(guarded_buffer&&) = delete; + + // copies @a input so its last byte is the last readable byte + const std::uint8_t* at_end(const std::string& input) + { + REQUIRE(input.size() <= page); + unsigned char* const first = base + (2 * page) - input.size(); + std::copy(input.begin(), input.end(), first); + return first; + } + + // copies @a input so its first byte is the first readable byte + const std::uint8_t* at_start(const std::string& input) + { + REQUIRE(input.size() <= page); + unsigned char* const first = base + page; + std::copy(input.begin(), input.end(), first); + return first; + } + + private: + static std::size_t page_size() + { +#if defined(JSON_TEST_GUARD_PAGE_WINDOWS) + SYSTEM_INFO info; + GetSystemInfo(&info); + return static_cast(info.dwPageSize); +#else + return static_cast(sysconf(_SC_PAGESIZE)); +#endif + } + + std::size_t page = 0; + unsigned char* base = nullptr; +}; + +// the serialization of the result of @a f, or the exception message +template +std::string outcome(F f) +{ + try + { + return "value: " + f().dump(-1, ' ', false, json::error_handler_t::keep); + } + catch (const json::exception& e) + { + return std::string("error: ") + e.what(); + } +} + +// a SAX handler that only counts events, so sax_parse runs without a DOM +struct counting_sax : nlohmann::json_sax +{ + std::size_t events = 0; + std::string error{}; // NOLINT(readability-redundant-member-init) + + bool null() override + { + return count(); + } + bool boolean(bool /*val*/) override + { + return count(); + } + bool number_integer(number_integer_t /*val*/) override + { + return count(); + } + bool number_unsigned(number_unsigned_t /*val*/) override + { + return count(); + } + bool number_float(number_float_t /*val*/, const string_t& /*s*/) override + { + return count(); + } + bool string(string_t& /*val*/) override + { + return count(); + } + bool binary(binary_t& /*val*/) override + { + return count(); + } + bool start_object(std::size_t /*elements*/) override + { + return count(); + } + bool key(string_t& /*val*/) override + { + return count(); + } + bool end_object() override + { + return count(); + } + bool start_array(std::size_t /*elements*/) override + { + return count(); + } + bool end_array() override + { + return count(); + } + bool parse_error(std::size_t position, const std::string& /*last_token*/, const nlohmann::detail::exception& ex) override + { + error = std::to_string(position) + " " + ex.what(); + return false; + } + + private: + bool count() + { + ++events; + return true; + } +}; + +// all prefixes of each seed, so the input ends at every byte of it +std::vector all_prefixes(const std::vector& seeds) +{ + std::vector result; + for (const auto& seed : seeds) + { + for (std::size_t n = 0; n <= seed.size(); ++n) + { + result.push_back(seed.substr(0, n)); + } + } + return result; +} + +std::vector json_seeds() +{ + std::vector seeds = + { + // string runs: ASCII, escapes, \u escapes, surrogate pairs + "\"abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ\"", + R"("ab\ncd\"ef\\gh\u00e4ij\uD83D\uDE00kl\u12")", + // 2-, 3- and 4-byte UTF-8 sequences, then invalid UTF-8 + std::string("\"\xC3\xA4\xE2\x82\xAC\xF0\x9F\x98\x80\xC3\xA4\xE2\x82\xAC\xF0\x9F\x98\x80") + "abcdefgh\"", + "\"abcdefgh\xC3\x28 \xED\xA0\x80 \xF4\x90\x80\x80 \xF8 \xC0\xAF\"", + std::string("\"abcdefg\x01") + "abcdefgh\"", + // numbers, so the 8-digit loop runs up to the end + "12345678", + "1234567890123456", + "12345678901234567890123", + "-0.000000001234567890123456789", + "3.14159265358979323846264338327950288e+100", + "-1.2345678901234567e-12", + "1E-5", + "[1.5,2e10,-3E-5,0.000000001,123456789012345678]", + // literals + "[true,false,null]", + // comments (only parsed with ignore_comments) + "/* comment */ [1, // line comment\n 2] /**/", + // a pretty-printed document + json::parse(R"({"a": [1, 2.5, "three", {"b": null, "c": [true, false]}], "d": {"e": {"f": {"g": "h"}}}})").dump(4), + json::parse(R"({"a": [1, 2.5, "three", {"b": null, "c": [true, false]}], "d": {"e": {"f": {"g": "h"}}}})").dump(8, '\t'), + }; + + // indentation runs of every length, with and without a closing bracket + for (std::size_t n = 0; n <= 20; ++n) + { + seeds.push_back("[\n" + std::string(n, ' ') + "1,\n" + std::string(n, ' ') + "2\n]"); + } + + return all_prefixes(seeds); +} + +// a value whose binary serializations contain long strings and keys, numbers +// of every width, and nested containers +json binary_document() +{ + json j; + j["a key that is longer than eight bytes"] = "a string value that is longer than eight bytes"; + j["\xC3\xA4\xE2\x82\xAC\xF0\x9F\x98\x80"] = "\xF0\x9F\x98\x80\xE2\x82\xAC\xC3\xA4 utf-8 0123456789"; + j["numbers"] = {0, 1, -1, 255, -256, 65536, -2147483649LL, 4294967296ULL, 1.5, -2.25e100}; + j["nested"] = {{"x", {{"y", {true, false, nullptr}}}}}; + j["s"] = ""; + return j; +} + +// readers for check_binary(), callable with a container or an iterator range +#define JSON_TEST_GUARD_PAGE_READER(name, from) \ + struct name \ + { \ + template \ + json operator()(Args&& ... args) const \ + { \ + return json::from(std::forward(args)...); \ + } \ + }; +JSON_TEST_GUARD_PAGE_READER(read_bson, from_bson) +JSON_TEST_GUARD_PAGE_READER(read_bon8, from_bon8) +JSON_TEST_GUARD_PAGE_READER(read_cbor, from_cbor) +JSON_TEST_GUARD_PAGE_READER(read_msgpack, from_msgpack) +JSON_TEST_GUARD_PAGE_READER(read_ubjson, from_ubjson) +JSON_TEST_GUARD_PAGE_READER(read_bjdata, from_bjdata) +#undef JSON_TEST_GUARD_PAGE_READER + +// reads every prefix of @a bytes at both ends of the guarded page and from a +// std::string, which must give the same outcome +template +void check_binary(guarded_buffer& guard, const std::vector& bytes, Reader read) +{ + for (const auto& input : all_prefixes({std::string(bytes.begin(), bytes.end())})) + { + CAPTURE(input) + const std::string expected = outcome([&] { return read(input); }); + for (int placement = 0; placement < 2; ++placement) + { + const std::uint8_t* const first = placement == 0 ? guard.at_end(input) : guard.at_start(input); + CHECK(outcome([&] { return read(first, first + input.size()); }) == expected); + } + } +} +} // namespace + +TEST_CASE("guard page") +{ + guarded_buffer guard; + + SECTION("string scanning") + { + // fills that drive each function to a different stop + const std::vector fills = + { + std::string(64, 'a'), + std::string(64, '"'), + std::string(64, '\\'), + std::string(64, '\x01'), + std::string(64, '\x7F'), + std::string(64, '\xFF'), + }; + std::vector patterns = fills; + { + std::string utf8; + while (utf8.size() < 64) + { + utf8 += "\xC3\xA4\xE2\x82\xAC\xF0\x9F\x98\x80" "ab"; + } + patterns.push_back(utf8); + } + { + std::string bytes; + for (unsigned int i = 0; i < 64; ++i) + { + bytes.push_back(static_cast((i * 37u) & 0xFFu)); + } + patterns.push_back(bytes); + } + + for (const auto& pattern : patterns) + { + for (std::size_t n = 0; n <= 64; ++n) + { + const std::string input = pattern.substr(0, n); + for (int placement = 0; placement < 2; ++placement) + { + const unsigned char* const p = placement == 0 ? guard.at_end(input) : guard.at_start(input); + + CHECK(nlohmann::detail::find_string_special(p, n) <= n); + CHECK(nlohmann::detail::find_ascii_copyable_run(p, n) <= n); + CHECK(nlohmann::detail::valid_utf8_prefix(p, n) <= n); +#if defined(JSON_USE_SIMDUTF) && defined(JSON_HAS_CPP_17) + CHECK(nlohmann::detail::find_string_delimiter(p, n) <= n); +#endif + CHECK(nlohmann::detail::string_bulk_run(p, n) <= n); + CHECK(nlohmann::detail::scalar_string_bulk_run(p, n) <= n); + for (std::size_t i = 0; i < n; ++i) + { + CHECK(nlohmann::detail::validate_one_utf8(p + i, n - i) <= n - i); + } + } + } + } + } + + SECTION("Eisel-Lemire") + { + const std::vector numbers = all_prefixes( + { + "1234567812345678123456781234567812345678", + "-1.2345678901234567890123456789e-123", + "0.00000000000000000000000000000000000001", + "98765432.10987654", + }); + for (const auto& number : numbers) + { + double out = 0; + const char* const end = reinterpret_cast(guard.at_end(number)); + static_cast(nlohmann::detail::parse_float_eisel_lemire(end, end + number.size(), out)); + const char* const start = reinterpret_cast(guard.at_start(number)); + static_cast(nlohmann::detail::parse_float_eisel_lemire(start, start + number.size(), out)); + } + } + + SECTION("JSON") + { + for (const auto& input : json_seeds()) + { + CAPTURE(input) + const std::string expected = outcome([&] { return json::parse(input); }); + const std::string expected_comments = outcome([&] { return json::parse(input, nullptr, true, true); }); + const bool expected_accept = json::accept(input); + counting_sax expected_sax; + const bool expected_sax_result = json::sax_parse(input, &expected_sax); + + for (int placement = 0; placement < 2; ++placement) + { + const std::uint8_t* const first = placement == 0 ? guard.at_end(input) : guard.at_start(input); + const std::uint8_t* const last = first + input.size(); + + CHECK(outcome([&] { return json::parse(first, last); }) == expected); + CHECK(outcome([&] { return json::parse(first, last, nullptr, true, true); }) == expected_comments); + CHECK(json::accept(first, last) == expected_accept); + + counting_sax sax; + CHECK(json::sax_parse(first, last, &sax) == expected_sax_result); + CHECK(sax.events == expected_sax.events); + CHECK(sax.error == expected_sax.error); + } + } + } + + SECTION("binary formats") + { + // every prefix ends inside a value, e.g., inside a BSON key or BON8 + // string without its terminator + const json j = binary_document(); + check_binary(guard, json::to_bson(j), read_bson()); + check_binary(guard, json::to_bon8(j), read_bon8()); + check_binary(guard, json::to_cbor(j), read_cbor()); + check_binary(guard, json::to_msgpack(j), read_msgpack()); + check_binary(guard, json::to_ubjson(j), read_ubjson()); + check_binary(guard, json::to_bjdata(j), read_bjdata()); + } +} + +#endif