Make parsing from std::istream and FILE* fast

Squashed onto develop cd6b428ef (CI fixes from #5860). Original commits:

- Let the lexer's bulk fast paths run on a window of the input
- Scan the get area of an input stream in bulk
- Lock a FILE once per parse instead of once per character
- Add benchmarks for parsing from a FILE* and for many values from a stream
- Avoid implicit unsigned char to char conversion in stream tests
- Make file_input_adapter::unlock() safe to call on a moved-from adapter

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

tests/CMakeLists.txt

@@ -285,6 +285,10 @@ if (JSON_TestSimdutf)
     endif()
 endif()
 
+# test-file_input checks from a second thread that parsing from a FILE* releases the file's lock
+find_package(Threads REQUIRED)
+json_test_set_test_options("test-file_input;test-file_input_unlocked_io_disabled" LINK_LIBRARIES Threads::Threads)
+
 # *DO* use json_test_set_test_options() above this line
 
 json_test_should_build_32bit_test(json_32bit_test json_32bit_test_only "${JSON_32bitTest}")
@@ -369,6 +373,33 @@ json_test_add_test_for(src/unit-diagnostic-positions.cpp
     MAIN test_main CXX_STANDARDS ${test_cxx_standards} ${test_force}
 )
 
+# test reading from a std::istream again without scanning the get area in bulk
+json_test_set_test_options(test-stream-bulk-scan_disabled
+    COMPILE_DEFINITIONS JSON_DISABLE_STREAM_BULK_SCAN=1
+)
+json_test_add_test_for(src/unit-stream-bulk-scan.cpp
+    NAME test-stream-bulk-scan_disabled
+    MAIN test_main CXX_STANDARDS ${test_cxx_standards} ${test_force}
+)
+
+# test reading from a std::istream again with JSON_PRECISE_STREAM_POSITION
+json_test_set_test_options(test-stream-bulk-scan_precise
+    COMPILE_DEFINITIONS JSON_PRECISE_STREAM_POSITION=1
+)
+json_test_add_test_for(src/unit-stream-bulk-scan.cpp
+    NAME test-stream-bulk-scan_precise
+    MAIN test_main CXX_STANDARDS ${test_cxx_standards} ${test_force}
+)
+
+# test reading from a FILE* again with the locking stdio functions
+json_test_set_test_options(test-file_input_unlocked_io_disabled
+    COMPILE_DEFINITIONS JSON_DISABLE_UNLOCKED_FILE_IO=1
+)
+json_test_add_test_for(src/unit-file_input.cpp
+    NAME test-file_input_unlocked_io_disabled
+    MAIN test_main CXX_STANDARDS ${test_cxx_standards} ${test_force}
+)
+
 endif()
 
 # *DO NOT* use json_test_set_test_options() below this line

tests/benchmarks/src/benchmarks.cpp

@@ -8,11 +8,13 @@
 
 #include <benchmark/benchmark.h>
 #include <nlohmann/json.hpp>
-#include <fstream>
 #include <array>
+#include <cstdio>
 #include <deque>
+#include <fstream>
 #include <numeric>
 #include <set>
+#include <sstream>
 #include <vector>
 #include <test_data.hpp>
 
@@ -51,6 +53,65 @@ BENCHMARK_CAPTURE(ParseFile, signed_ints,       TEST_DATA_DIRECTORY "/regression
 BENCHMARK_CAPTURE(ParseFile, unsigned_ints,     TEST_DATA_DIRECTORY "/regression/unsigned_ints.json");
 BENCHMARK_CAPTURE(ParseFile, small_signed_ints, TEST_DATA_DIRECTORY "/regression/small_signed_ints.json");
 
+//////////////////////////////////////////////////////////////////////////////
+// parse JSON from a FILE*
+//////////////////////////////////////////////////////////////////////////////
+
+static void ParseFILE(benchmark::State& state, const char* filename)
+{
+    while (state.KeepRunning())
+    {
+        state.PauseTiming();
+        std::FILE* f = std::fopen(filename, "rb");
+        auto* j = new json();
+        state.ResumeTiming();
+
+        *j = json::parse(f);
+
+        state.PauseTiming();
+        std::fclose(f);
+        delete j;
+        state.ResumeTiming();
+    }
+
+    std::ifstream file(filename, std::ios::binary | std::ios::ate);
+    state.SetBytesProcessed(state.iterations() * file.tellg());
+}
+BENCHMARK_CAPTURE(ParseFILE, canada,       TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json");
+BENCHMARK_CAPTURE(ParseFILE, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
+BENCHMARK_CAPTURE(ParseFILE, twitter,      TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
+
+//////////////////////////////////////////////////////////////////////////////
+// read many small values from one stream (NDJSON)
+//////////////////////////////////////////////////////////////////////////////
+
+static void ParseStreamValues(benchmark::State& state, bool numbers)
+{
+    std::string input;
+    for (int i = 0; i < 10000; ++i)
+    {
+        input += numbers ? std::to_string(i * 7919) + " "
+                 : R"({"id":)" + std::to_string(i) + R"(,"name":"user)" + std::to_string(i) + R"(","active":true,"score":)" + std::to_string(i * 0.25) + "}\n";
+    }
+
+    for (auto _ : state)
+    {
+        std::istringstream is(input);
+        json j;
+        std::size_t count = 0;
+        while (!(is >> std::ws).eof())
+        {
+            is >> j;
+            ++count;
+        }
+        benchmark::DoNotOptimize(count);
+    }
+
+    state.SetBytesProcessed(static_cast<int64_t>(state.iterations() * input.size()));
+}
+BENCHMARK_CAPTURE(ParseStreamValues, ndjson,  false);
+BENCHMARK_CAPTURE(ParseStreamValues, numbers, true);
+
 //////////////////////////////////////////////////////////////////////////////
 // parse JSON from string
 //////////////////////////////////////////////////////////////////////////////

tests/src/unit-file_input.cpp

@@ -0,0 +1,236 @@
+//     __ _____ _____ _____
+//  __|  |   __|     |   | |  JSON for Modern C++ (supporting code)
+// |  |  |__   |  |  | | | |  version 3.12.0
+// |_____|_____|_____|_|___|  https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+#include "doctest_compatibility.h"
+
+#include <nlohmann/json.hpp>
+using nlohmann::json;
+
+#include <array>
+#include <cstdint>
+#include <cstdio>
+#include <fstream>
+#include <memory>
+#include <random>
+#include <string>
+#include <thread>
+#include <vector>
+
+// Parsing from a FILE* locks the file once and reads it with the unlocked
+// stdio functions (see JSON_DISABLE_UNLOCKED_FILE_IO). These tests pin that
+// this is not observable: the results are the same, the file is unlocked
+// again when parsing ends - also with an exception - and the same thread can
+// keep using the file while it is locked, e.g., from a SAX callback.
+
+namespace
+{
+// a file name of its own for every run: the test runs for several C++
+// standards and configurations at the same time, in the same directory
+std::string temp_file_name()
+{
+    std::random_device rd;
+    return "unit-file_input-" + std::to_string(rd()) + ".tmp";
+}
+
+using file_ptr = std::unique_ptr<std::FILE, int (*)(std::FILE*)>;
+
+// write @a content to the file @a name and open it for reading
+file_ptr open_with(const char* name, const std::string& content)
+{
+    {
+        std::ofstream file(name, std::ios::binary);
+        file << content;
+    }
+    return {std::fopen(name, "rb"), &std::fclose};
+}
+
+// The stdio lock is recursive, so whether parsing released it can only be
+// seen from another thread: ftrylockfile() fails there while this thread
+// holds the lock. ftrylockfile() is POSIX; elsewhere, the check is skipped.
+#if defined(__GLIBC__) || defined(__APPLE__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__)
+void check_unlocked(std::FILE* f)
+{
+    int result = -1;
+    std::thread other([f, &result]
+    {
+        result = ftrylockfile(f);
+        if (result == 0)
+        {
+            funlockfile(f);
+        }
+    });
+    other.join();
+    CHECK(result == 0);
+}
+#else
+void check_unlocked(std::FILE* /*f*/) {}
+#endif
+
+// SAX consumer that also reads the position of the file it is parsing, which
+// takes the file's lock again on the same thread
+class position_reading_sax : public nlohmann::json_sax<json>
+{
+  public:
+    explicit position_reading_sax(std::FILE* f) : m_file(f) {}
+    position_reading_sax(const position_reading_sax&) = delete;
+    position_reading_sax(position_reading_sax&&) = delete;
+    position_reading_sax& operator=(const position_reading_sax&) = delete;
+    position_reading_sax& operator=(position_reading_sax&&) = delete;
+    ~position_reading_sax() override = default;
+
+    bool null() override
+    {
+        return record();
+    }
+    bool boolean(bool /*val*/) override
+    {
+        return record();
+    }
+    bool number_integer(number_integer_t /*val*/) override
+    {
+        return record();
+    }
+    bool number_unsigned(number_unsigned_t /*val*/) override
+    {
+        return record();
+    }
+    bool number_float(number_float_t /*val*/, const string_t& /*s*/) override
+    {
+        return record();
+    }
+    bool string(string_t& /*val*/) override
+    {
+        return record();
+    }
+    bool binary(binary_t& /*val*/) override
+    {
+        return record();
+    }
+    bool start_object(std::size_t /*elements*/) override
+    {
+        return record();
+    }
+    bool key(string_t& /*val*/) override
+    {
+        return record();
+    }
+    bool end_object() override
+    {
+        return record();
+    }
+    bool start_array(std::size_t /*elements*/) override
+    {
+        return record();
+    }
+    bool end_array() override
+    {
+        return record();
+    }
+    bool parse_error(std::size_t /*position*/, const std::string& /*last_token*/, const nlohmann::detail::exception& /*ex*/) override
+    {
+        return false;
+    }
+
+    const std::vector<long>& positions() const
+    {
+        return m_positions;
+    }
+
+  private:
+    bool record()
+    {
+        m_positions.push_back(std::ftell(m_file));
+        return true;
+    }
+
+    std::FILE* m_file;
+    std::vector<long> m_positions{}; // NOLINT(readability-redundant-member-init)
+};
+}  // namespace
+
+TEST_CASE("parsing from a FILE*")
+{
+    const std::string temp_name = temp_file_name();
+    const char* const name = temp_name.c_str();
+    const std::string input = "{\"a\": [1, -2.5, \"three\", true, null], \"b\": {\"c\": \"caf\xc3\xa9\"}}";
+
+    SECTION("values are the same as when parsing from a string")
+    {
+        const file_ptr f = open_with(name, input);
+        REQUIRE(f != nullptr);
+        CHECK(json::parse(f.get()) == json::parse(input));
+        // the whole input was consumed
+        std::array<char, 8> rest{};
+        CHECK(std::fgets(rest.data(), static_cast<int>(rest.size()), f.get()) == nullptr);
+        check_unlocked(f.get());
+    }
+
+    SECTION("the file is unlocked after a parse error")
+    {
+        const file_ptr f = open_with(name, "[1, 2, x]");
+        REQUIRE(f != nullptr);
+        json _;
+        CHECK_THROWS_WITH_AS(_ = json::parse(f.get()),
+                             "[json.exception.parse_error.101] parse error at line 1, column 8: syntax error while parsing value - invalid literal; last read: '2, x'",
+                             json::parse_error&);
+        check_unlocked(f.get());
+
+        // the file is still usable: the rest of the input is there
+        std::array<char, 8> rest{};
+        CHECK(std::fgets(rest.data(), static_cast<int>(rest.size()), f.get()) != nullptr);
+        CHECK(std::string(rest.data()) == "]");
+    }
+
+    SECTION("the file can be parsed again and read with stdio afterwards")
+    {
+        const file_ptr f = open_with(name, "[1] [2] tail");
+        REQUIRE(f != nullptr);
+
+        // sax_parse with strict == false stops after the first value; an array
+        // ends with its closing bracket, so the space after it is not read
+        json j;
+        nlohmann::detail::json_sax_dom_parser<json, nlohmann::detail::string_input_adapter_type> sax1(j);
+        CHECK(json::sax_parse(f.get(), &sax1, json::input_format_t::json, false));
+        CHECK(j == json({1}));
+        check_unlocked(f.get());
+
+        json k;
+        nlohmann::detail::json_sax_dom_parser<json, nlohmann::detail::string_input_adapter_type> sax2(k);
+        CHECK(json::sax_parse(f.get(), &sax2, json::input_format_t::json, false));
+        CHECK(k == json({2}));
+        check_unlocked(f.get());
+
+        std::array<char, 8> rest{};
+        CHECK(std::fgets(rest.data(), static_cast<int>(rest.size()), f.get()) != nullptr);
+        CHECK(std::string(rest.data()) == " tail");
+    }
+
+    SECTION("the same thread can use the file while it is parsed")
+    {
+        const file_ptr f = open_with(name, "[10, 20, 30]");
+        REQUIRE(f != nullptr);
+        position_reading_sax sax(f.get());
+        CHECK(json::sax_parse(f.get(), &sax));
+        // start_array, three numbers, end_array
+        CHECK(sax.positions().size() == 5);
+        CHECK(sax.positions().front() == 1);
+        check_unlocked(f.get());
+    }
+
+    SECTION("binary formats read from a FILE*")
+    {
+        const json j = {{"string", std::string(300, 'x')}, {"array", {1, 2, 3}}, {"number", 1.5}};
+        const std::vector<std::uint8_t> cbor = json::to_cbor(j);
+        const file_ptr f = open_with(name, std::string(reinterpret_cast<const char*>(cbor.data()), cbor.size()));
+        REQUIRE(f != nullptr);
+        CHECK(json::from_cbor(f.get()) == j);
+        check_unlocked(f.get());
+    }
+
+    static_cast<void>(std::remove(name));
+}

tests/src/unit-precise-stream-position.cpp

@@ -20,6 +20,7 @@
 #include <nlohmann/json.hpp>
 using nlohmann::json;
 
+#include <algorithm>
 #include <cstddef>
 #include <sstream>
 #include <streambuf>
@@ -72,6 +73,40 @@ class no_putback_streambuf : public std::streambuf
     std::size_t m_pos = 0;
 };
 
+// A streambuf that serves its input in get areas of a fixed size, so that the
+// end of a number falls on every possible offset of a get area. The lexer scans
+// the get area directly; the stream position must not depend on its size.
+class chunked_streambuf : public std::streambuf
+{
+  public:
+    chunked_streambuf(std::string s, std::size_t chunk_size)
+        : m_data(std::move(s)), m_chunk_size(chunk_size)
+    {}
+
+  protected:
+    int_type underflow() override
+    {
+        if (gptr() < egptr())
+        {
+            return traits_type::to_int_type(*gptr());
+        }
+        m_pos += static_cast<std::size_t>(egptr() - eback());
+        if (m_pos >= m_data.size())
+        {
+            setg(nullptr, nullptr, nullptr);
+            return traits_type::eof();
+        }
+        char* const begin = &m_data[m_pos];
+        setg(begin, begin, begin + (std::min)(m_chunk_size, m_data.size() - m_pos));
+        return traits_type::to_int_type(*gptr());
+    }
+
+  private:
+    std::string m_data;
+    std::size_t m_pos = 0;
+    std::size_t m_chunk_size;
+};
+
 // read the characters that are left in a stream
 std::string remaining(std::istream& is)
 {
@@ -263,4 +298,26 @@ TEST_CASE("JSON_PRECISE_STREAM_POSITION")
         json _;
         CHECK_THROWS_WITH_AS(_ = json::parse(is), "[json.exception.parse_error.101] parse error: attempting to parse an empty input; check that your input string or stream contains the expected JSON", json::parse_error&);
     }
+
+    SECTION("the position does not depend on the get area of the streambuf")
+    {
+        const std::vector<std::size_t> chunk_sizes = {1, 2, 3, 4, 5, 7, 8, 13, 4096};
+        for (const std::size_t chunk_size : chunk_sizes)
+        {
+            CAPTURE(chunk_size)
+            chunked_streambuf buf("12345678901234567890true\"a long string value\"-1.25e+3,[1,2]42", chunk_size);
+            std::istream input(&buf);
+            json j;
+
+            input >> j;
+            CHECK(j == 12345678901234567890u);
+            input >> j;
+            CHECK(j == true);
+            input >> j;
+            CHECK(j == "a long string value");
+            input >> j;
+            CHECK(j == -1250.0);
+            CHECK(remaining(input) == ",[1,2]42");
+        }
+    }
 }

tests/src/unit-stream-bulk-scan.cpp

@@ -0,0 +1,417 @@
+//     __ _____ _____ _____
+//  __|  |   __|     |   | |  JSON for Modern C++ (supporting code)
+// |  |  |__   |  |  | | | |  version 3.12.0
+// |_____|_____|_____|_|___|  https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+#include "doctest_compatibility.h"
+
+// the library undefines its configuration macros at the end of the header
+#if defined(JSON_DISABLE_STREAM_BULK_SCAN) && JSON_DISABLE_STREAM_BULK_SCAN
+    #define TEST_STREAM_BULK_SCAN 0
+#else
+    #define TEST_STREAM_BULK_SCAN 1
+#endif
+
+#include <nlohmann/json.hpp>
+using nlohmann::json;
+
+#include <algorithm>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <fstream>
+#include <istream>
+#include <random>
+#include <sstream>
+#include <streambuf>
+#include <string>
+#include <utility>
+#include <vector>
+
+// The lexer scans the characters in the get area of a std::istream's
+// streambuf directly instead of reading them one at a time. These tests pin
+// that this is not observable: for any get area size - including a streambuf
+// without a get area, which still takes the character-at-a-time path - the
+// parsed values, the exception messages (including the "last read" token),
+// the stream state, and the characters left in the stream are the same. See
+// unit-stream-bulk-scan-disabled.cpp for JSON_DISABLE_STREAM_BULK_SCAN.
+
+namespace
+{
+// A streambuf that serves its input in get areas of a fixed size, so that
+// tokens, numbers, UTF-8 sequences, and indentation cross the end of the get
+// area at every possible offset. With a size of 0, it keeps no get area at all
+// and serves every character from underflow()/uflow().
+class chunked_streambuf : public std::streambuf
+{
+  public:
+    chunked_streambuf(std::string s, std::size_t chunk_size)
+        : m_data(std::move(s)), m_chunk_size(chunk_size)
+    {}
+
+  protected:
+    int_type underflow() override
+    {
+        if (m_chunk_size == 0)
+        {
+            return m_pos < m_data.size() ? traits_type::to_int_type(m_data[m_pos]) : traits_type::eof();
+        }
+        if (gptr() < egptr())
+        {
+            return traits_type::to_int_type(*gptr());
+        }
+        // the previous get area has been consumed
+        m_pos += static_cast<std::size_t>(egptr() - eback());
+        if (m_pos >= m_data.size())
+        {
+            setg(nullptr, nullptr, nullptr);
+            return traits_type::eof();
+        }
+        char* const begin = &m_data[m_pos];
+        setg(begin, begin, begin + (std::min)(m_chunk_size, m_data.size() - m_pos));
+        return traits_type::to_int_type(*gptr());
+    }
+
+    int_type uflow() override
+    {
+        if (m_chunk_size == 0)
+        {
+            return m_pos < m_data.size() ? traits_type::to_int_type(m_data[m_pos++]) : traits_type::eof();
+        }
+        return std::streambuf::uflow();
+    }
+
+  private:
+    std::string m_data;
+    std::size_t m_pos = 0;
+    std::size_t m_chunk_size;
+};
+
+// how a test reads from the stream
+enum class read_mode
+{
+    extract,         // up to four times operator>>, as for concatenated values
+    parse,           // json::parse(stream)
+    parse_lenient,   // json::parse(stream) with comments
+};
+
+// Read @a input from a chunked_streambuf and describe everything observable:
+// the values or the exception, the stream state after each value, and the
+// characters left in the stream.
+std::string observe(const std::string& input, std::size_t chunk_size, read_mode mode)
+{
+    chunked_streambuf buf(input, chunk_size);
+    std::istream is(&buf);
+    std::string result;
+    for (int i = 0; i < 4; ++i)
+    {
+        json j;
+        try
+        {
+            if (mode == read_mode::extract)
+            {
+                is >> j;
+            }
+            else
+            {
+                j = json::parse(is, nullptr, true, mode == read_mode::parse_lenient);
+            }
+            result += "value " + j.dump() + "; state " + std::to_string(static_cast<unsigned int>(is.rdstate())) + "\n";
+        }
+        catch (const json::exception& e)
+        {
+            result += std::string("exception ") + e.what() + "\n";
+            break;
+        }
+        if (mode != read_mode::extract)
+        {
+            break;
+        }
+        is.clear();
+    }
+
+    is.clear();
+    result += "remaining ";
+    char c = 0;
+    while (is.get(c))
+    {
+        result += c;
+    }
+    return result;
+}
+
+// get area sizes that put every boundary everywhere in short inputs, plus one
+// that holds most inputs completely
+std::vector<std::size_t> chunk_sizes()
+{
+    return {1, 2, 3, 4, 5, 7, 8, 9, 13, 16, 64, 4096};
+}
+
+// valid and invalid inputs covering every token type and every bulk path:
+// strings (plain runs, escapes, UTF-8), numbers (all grammar parts, too large
+// for an integer), and indentation
+std::vector<std::string> corpus()
+{
+    return
+    {
+        // numbers, terminated in every way
+        "0", "-0", "123", "-123", "1.5", "-1.5e+10", "2E-3", "123456789012345678901234567890",
+        "18446744073709551615", "18446744073709551616", "-9223372036854775809",
+        "123 456", "123,456", "123]", "1true", "42\nfoo", "1.25e3}", "7\t8\r9",
+        // malformed numbers
+        "1.", "-", "1e", "1e+", "01", "-a", "1.x", "1ee5",
+        // strings
+        R"("")", R"("abc")", R"("a long string with a run of ordinary characters to scan in bulk")",
+        R"("escapes \" \\ \/ \b \f \n \r \t é 😀")",
+        "\"UTF-8: \xc3\xa9 \xe2\x82\xac \xf0\x9f\x98\x80 done\"",
+        // malformed strings
+        R"("unterminated)", "\"control\x01character\"", "\"ill-formed \xc3\x28 UTF-8\"", R"("bad \x escape")",
+        R"("\ud800 lone surrogate")",
+        // literals
+        "true", "false", "null", "tru", "nul", "falsey",
+        // structures
+        "[]", "{}", "[1,2,3]", R"({"a":1,"b":[true,false,null],"c":{"d":"e"}})", "[1,]", R"({"a" 1})", "[[[[[]]]]]",
+        // pretty-printed
+        "{\n    \"key\": [\n        1,\n        -2.5,\n        \"three\"\n    ]\n}",
+        "[\n                                \"deep\"\n                                ]",
+        "{\n    \"a\": 1,\n    \"b\": 2x\n}",
+        // concatenated values and whitespace
+        "1 2 3", R"("a""b")", "[1][2]", "{}{}", " \n\t 42 \n\t ", "",
+        // comments (only accepted by parse_lenient)
+        "/* comment */ 1", "// comment\n[1, /* c */ 2]",
+        // byte order mark
+        "\xef\xbb\xbf[1]",
+    };
+}
+}  // namespace
+
+TEST_CASE("parsing from a std::istream scans the get area in bulk")
+{
+    SECTION("results do not depend on the get area")
+    {
+        const std::vector<read_mode> read_modes = {read_mode::extract, read_mode::parse, read_mode::parse_lenient};
+        for (const auto& input : corpus())
+        {
+            for (const read_mode mode : read_modes)
+            {
+                // no get area: the character-at-a-time path
+                const std::string expected = observe(input, 0, mode);
+                for (const std::size_t chunk_size : chunk_sizes())
+                {
+                    CAPTURE(input)
+                    CAPTURE(chunk_size)
+                    CAPTURE(static_cast<int>(mode))
+                    CHECK(observe(input, chunk_size, mode) == expected);
+                }
+            }
+        }
+    }
+
+    SECTION("values are the same as when parsing from a string")
+    {
+        const std::string input = R"({"name": "nlohmann::json", "numbers": [1, -2, 3.5, 4e10, 18446744073709551615],)"
+                                  "\n    \"text\": \"caf\xc3\xa9 \\u00e9 \\\"quoted\\\"\", \"nested\": [[{}], [[]]], \"flags\": [true, false, null]}";
+        const json expected = json::parse(input);
+        for (const std::size_t chunk_size : chunk_sizes())
+        {
+            CAPTURE(chunk_size)
+            chunked_streambuf buf(input, chunk_size);
+            std::istream is(&buf);
+            CHECK(json::parse(is) == expected);
+        }
+
+        std::istringstream ss(input);
+        CHECK(json::parse(ss) == expected);
+    }
+
+    SECTION("a token that spans several get areas is reported completely")
+    {
+        // the error is found after the string has crossed many get areas; the
+        // token string holds everything since the start of the last token
+        const std::string input = R"(["this string is longer than every get area", tru])";
+        for (const std::size_t chunk_size : chunk_sizes())
+        {
+            CAPTURE(chunk_size)
+            chunked_streambuf buf(input, chunk_size);
+            std::istream is(&buf);
+            json j;
+            CHECK_THROWS_WITH_AS(is >> j,
+                                 "[json.exception.parse_error.101] parse error at line 1, column 50: syntax error while parsing value - invalid literal; last read: '\"this string is longer than every get area\", tru]'",
+                                 json::parse_error&);
+        }
+
+        const std::string unterminated = R"("this string is longer than every get area)";
+        for (const std::size_t chunk_size : chunk_sizes())
+        {
+            CAPTURE(chunk_size)
+            chunked_streambuf buf(unterminated, chunk_size);
+            std::istream is(&buf);
+            json j;
+            CHECK_THROWS_WITH_AS(is >> j,
+                                 "[json.exception.parse_error.101] parse error at line 1, column 43: syntax error while parsing value - invalid string: missing closing quote; last read: '\"this string is longer than every get area'",
+                                 json::parse_error&);
+        }
+    }
+
+    SECTION("an error before the first token reports everything read")
+    {
+        chunked_streambuf buf("  \n  x", 2);
+        std::istream is(&buf);
+        json j;
+        CHECK_THROWS_WITH_AS(is >> j,
+                             "[json.exception.parse_error.101] parse error at line 2, column 3: syntax error while parsing value - invalid literal; last read: '  <U+000A>  x'",
+                             json::parse_error&);
+    }
+
+    SECTION("lines and columns are counted across get areas")
+    {
+        const std::string input = "[\n    1,\n    \"two\",\n    3.0,\n    x\n]";
+        for (const std::size_t chunk_size : chunk_sizes())
+        {
+            CAPTURE(chunk_size)
+            chunked_streambuf buf(input, chunk_size);
+            std::istream is(&buf);
+            json j;
+            CHECK_THROWS_WITH_AS(is >> j,
+                                 "[json.exception.parse_error.101] parse error at line 5, column 5: syntax error while parsing value - invalid literal; last read: '3.0,<U+000A>    x'",
+                                 json::parse_error&);
+        }
+    }
+
+    SECTION("many values from one stream")
+    {
+        std::string input;
+        for (int i = 0; i < 200; ++i)
+        {
+            input += R"({"id":)" + std::to_string(i) + R"(,"name":"user)" + std::to_string(i) + R"(","score":)" + std::to_string(i) + ".5}\n";
+            input += std::to_string(i * 7919) + "\n";
+        }
+
+        for (const std::size_t chunk_size : chunk_sizes())
+        {
+            CAPTURE(chunk_size)
+            chunked_streambuf buf(input, chunk_size);
+            std::istream is(&buf);
+            json j;
+            int count = 0;
+            while (!(is >> std::ws).eof())
+            {
+                is >> j;
+                if (count % 2 == 0)
+                {
+                    CHECK(j["id"] == count / 2);
+                }
+                else
+                {
+                    CHECK(j == (count / 2) * 7919);
+                }
+                ++count;
+            }
+            CHECK(count == 400);
+        }
+    }
+
+    SECTION("the stream can be used after parsing")
+    {
+        for (const std::size_t chunk_size : chunk_sizes())
+        {
+            CAPTURE(chunk_size)
+            chunked_streambuf buf(R"({"a": "some text"} trailing words 42)", chunk_size);
+            std::istream is(&buf);
+            json j;
+            is >> j;
+            CHECK(j == json({{"a", "some text"}}));
+
+            std::string word;
+            int number = 0;
+            is >> word;
+            CHECK(word == "trailing");
+            is >> word >> number;
+            CHECK(word == "words");
+            CHECK(number == 42);
+        }
+    }
+
+    SECTION("std::stringstream and std::ifstream")
+    {
+        const std::string input = R"([1, 2.5, "three", {"four": [true, null]}] 5)";
+
+        std::stringstream ss(input);
+        json j1;
+        json j2;
+        ss >> j1 >> j2;
+        CHECK(j1 == json::parse(R"([1, 2.5, "three", {"four": [true, null]}])"));
+        CHECK(j2 == 5);
+        CHECK(ss.eof());
+
+        // a file name of its own for every run: the test runs for several C++
+        // standards and configurations at the same time, in the same directory
+        std::random_device rd;
+        const std::string temp_name = "unit-stream-bulk-scan-" + std::to_string(rd()) + ".tmp";
+        const char* const filename = temp_name.c_str();
+        {
+            std::ofstream file(filename);
+            file << input;
+        }
+        {
+            std::ifstream file(filename);
+            json k1;
+            json k2;
+            file >> k1 >> k2;
+            CHECK(k1 == j1);
+            CHECK(k2 == j2);
+        }
+        static_cast<void>(std::remove(filename));
+    }
+
+    SECTION("binary formats read from a stream")
+    {
+        const json j = {{"string", std::string(300, 'x')}, {"array", {1, 2, 3}}, {"number", 1.5}};
+        const std::vector<std::uint8_t> cbor = json::to_cbor(j);
+        const std::string bytes(reinterpret_cast<const char*>(cbor.data()), cbor.size());
+        for (const std::size_t chunk_size : chunk_sizes())
+        {
+            CAPTURE(chunk_size)
+            chunked_streambuf buf(bytes, chunk_size);
+            std::istream is(&buf);
+            CHECK(json::from_cbor(is) == j);
+        }
+    }
+
+#if TEST_STREAM_BULK_SCAN
+    SECTION("the adapter keeps no token unless the lexer starts one")
+    {
+        // the binary readers read through the same adapter, but never start a
+        // token; refilling the get area must then not collect the input
+        chunked_streambuf buf(std::string(1000, 'x'), 7);
+        std::istream is(&buf);
+        nlohmann::detail::input_stream_adapter ia(is);
+        for (int i = 0; i < 1000; ++i)
+        {
+            CHECK(ia.get_character() == 'x');
+        }
+
+        std::vector<char> token;
+        ia.copy_token(token, false);
+        CHECK(token.empty());
+    }
+#endif
+
+    SECTION("SAX parsing from a stream")
+    {
+        const std::string input = R"({"a": [1, "two", 3.5]} rest)";
+        for (const std::size_t chunk_size : chunk_sizes())
+        {
+            CAPTURE(chunk_size)
+            chunked_streambuf buf(input, chunk_size);
+            std::istream is(&buf);
+            json j;
+            nlohmann::detail::json_sax_dom_parser<json, nlohmann::detail::string_input_adapter_type> sax(j);
+            CHECK(json::sax_parse(is, &sax, json::input_format_t::json, false));
+            CHECK(j == json::parse(R"({"a": [1, "two", 3.5]})"));
+        }
+    }
+}