diff --git a/docs/mkdocs/docs/api/basic_json/accept.md b/docs/mkdocs/docs/api/basic_json/accept.md index 89111f69d..4abf5d36e 100644 --- a/docs/mkdocs/docs/api/basic_json/accept.md +++ b/docs/mkdocs/docs/api/basic_json/accept.md @@ -132,6 +132,8 @@ outside of a string, invalid) byte; see the [FAQ entry](../../home/faq.md#nul-by - [operator>>](../operator_gtgt.md) - deserialize from stream - [`JSON_STRICT_NUL_HANDLING`](../macros/json_strict_nul_handling.md) - opt in to rejecting a NUL byte in the input instead of treating it as end of input +- [`JSON_DISABLE_STREAM_BULK_SCAN`](../macros/json_disable_stream_bulk_scan.md) - read a `std::istream` one character + at a time ## Version history @@ -143,6 +145,8 @@ outside of a string, invalid) byte; see the [FAQ entry](../../home/faq.md#nul-by - Extended overload (2) to accept heterogeneous iterator+sentinel pairs (C++20 ranges support) in version 3.13.0. - `JSON_STRICT_NUL_HANDLING` added in version 3.13.0 to optionally reject a NUL byte in the input instead of treating it as end of input; planned to become the default in version 4.0.0. +- Parsing from an `std::istream` scans the characters in its stream buffer in bulk in version 3.13.0 + ([`JSON_DISABLE_STREAM_BULK_SCAN`](../macros/json_disable_stream_bulk_scan.md)). !!! warning "Deprecation" diff --git a/docs/mkdocs/docs/api/basic_json/parse.md b/docs/mkdocs/docs/api/basic_json/parse.md index c64f93a87..7ac05b94b 100644 --- a/docs/mkdocs/docs/api/basic_json/parse.md +++ b/docs/mkdocs/docs/api/basic_json/parse.md @@ -112,6 +112,14 @@ By default, a `'\0'` (NUL) byte anywhere in the input is treated as end of input outside of a string, invalid) byte; see the [FAQ entry](../../home/faq.md#nul-bytes-in-the-input) for details and the [`JSON_STRICT_NUL_HANDLING`](../macros/json_strict_nul_handling.md) macro to opt into rejecting it instead. +!!! tip "Performance" + + Contiguous inputs (strings, string views, vectors, arrays, pointer ranges) are the fastest to parse. A + `#!cpp std::istream` comes close: the parser scans the characters its stream buffer holds in bulk, without reading + ahead, so the stream is left exactly where it would be after reading one character at a time. See + [input sources and performance](../../features/parsing/index.md#input-sources-and-performance) and + [`JSON_DISABLE_STREAM_BULK_SCAN`](../macros/json_disable_stream_bulk_scan.md). + ## Examples ??? example "Example: (1) parse from a character array" @@ -247,6 +255,8 @@ outside of a string, invalid) byte; see the [FAQ entry](../../home/faq.md#nul-by - [operator>>](../operator_gtgt.md) - deserialize from stream - [`JSON_STRICT_NUL_HANDLING`](../macros/json_strict_nul_handling.md) - opt in to rejecting a NUL byte in the input instead of treating it as end of input +- [`JSON_DISABLE_STREAM_BULK_SCAN`](../macros/json_disable_stream_bulk_scan.md) - read a `std::istream` one character + at a time ## Version history @@ -261,6 +271,9 @@ outside of a string, invalid) byte; see the [FAQ entry](../../home/faq.md#nul-by it as end of input; planned to become the default in version 4.0.0. - Extended empty-input detection to also cover an `std::istream` without a stream buffer, and fixed a crash (`std::terminate`) when parsing from an `std::istream` with `eofbit` in its exception mask, in version 3.13.0. +- Parsing from an `std::istream` scans the characters in its stream buffer in bulk in version 3.13.0 + ([`JSON_DISABLE_STREAM_BULK_SCAN`](../macros/json_disable_stream_bulk_scan.md) restores reading one character at a + time). !!! warning "Deprecation" diff --git a/docs/mkdocs/docs/api/basic_json/sax_parse.md b/docs/mkdocs/docs/api/basic_json/sax_parse.md index fc48c9e12..811cea6ff 100644 --- a/docs/mkdocs/docs/api/basic_json/sax_parse.md +++ b/docs/mkdocs/docs/api/basic_json/sax_parse.md @@ -137,6 +137,8 @@ A UTF-8 byte order mark is silently ignored. - [parse](parse.md) - deserialize from a compatible input - [accept](accept.md) - check if the input is valid JSON +- [`JSON_DISABLE_STREAM_BULK_SCAN`](../macros/json_disable_stream_bulk_scan.md) - read a `std::istream` one character + at a time ## Version history @@ -148,6 +150,8 @@ A UTF-8 byte order mark is silently ignored. - Extended overload (2) to accept heterogeneous iterator+sentinel pairs (C++20 ranges support) in version 3.13.0. - `JSON_PRECISE_STREAM_POSITION` added in version 3.13.0 to optionally leave a `#!cpp std::istream` positioned right after the parsed value when `strict` is `#!cpp false`. +- Parsing from an `std::istream` scans the characters in its stream buffer in bulk in version 3.13.0 + ([`JSON_DISABLE_STREAM_BULK_SCAN`](../macros/json_disable_stream_bulk_scan.md)). !!! warning "Deprecation" diff --git a/docs/mkdocs/docs/api/macros/index.md b/docs/mkdocs/docs/api/macros/index.md index be9f6332c..149262989 100644 --- a/docs/mkdocs/docs/api/macros/index.md +++ b/docs/mkdocs/docs/api/macros/index.md @@ -16,6 +16,8 @@ header. See also the [macro overview page](../../features/macros.md). ## Parsing +- [**JSON_DISABLE_STREAM_BULK_SCAN**](json_disable_stream_bulk_scan.md) - read a `std::istream` one character at a time + instead of scanning its stream buffer in bulk - [**JSON_PRECISE_STREAM_POSITION**](json_precise_stream_position.md) - opt in to leaving an input stream positioned right after a parsed number - [**JSON_STRICT_BINARY_UTF8**](json_strict_binary_utf8.md) - opt in to checking strings for valid UTF-8 in the CBOR, diff --git a/docs/mkdocs/docs/api/macros/json_disable_stream_bulk_scan.md b/docs/mkdocs/docs/api/macros/json_disable_stream_bulk_scan.md new file mode 100644 index 000000000..5a765061c --- /dev/null +++ b/docs/mkdocs/docs/api/macros/json_disable_stream_bulk_scan.md @@ -0,0 +1,142 @@ +# JSON_DISABLE_STREAM_BULK_SCAN + +```cpp +#define JSON_DISABLE_STREAM_BULK_SCAN /* value */ +``` + +When defined to `1`, parsing from a `#!cpp std::istream` reads the input one character at a time through the stream +buffer, as in version 3.12.0 and earlier, instead of scanning the characters that the stream buffer already holds in +bulk. + +The macro is an escape hatch for stream buffers that do not follow the usual contract of `std::basic_streambuf` (see +the requirements on the stream buffer in the notes below). It does not change any result: values, exceptions, and the +position of the stream after parsing are the same either way. Only the speed differs. + +## Default definition + +The default value is `0`: streams are scanned in bulk. + +```cpp +#define JSON_DISABLE_STREAM_BULK_SCAN 0 +``` + +## Notes + +!!! note "How streams are scanned" + + A `std::streambuf` keeps the part of the input it has read but not yet handed out in its *get area*, the range + from `gptr()` to `egptr()`. A `std::filebuf`, for instance, reads the file in blocks of a few kilobytes into its get + area, and a `std::stringbuf` exposes its whole string as one get area. + + When parsing from a `std::istream`, the parser scans this get area directly, with the same fast paths that it uses + for a `std::string`: runs of ordinary string characters, numbers, and indentation are recognized several bytes at a + time instead of being requested from the stream buffer one character at a time. When the parser reaches the end of + the get area, it asks the stream buffer for the next character as before, which makes the stream buffer refill its + get area. + + The parser only reads characters that it would also have read one at a time, and it advances the stream buffer + exactly over the characters it consumes. Nothing is read ahead, buffered elsewhere, or put back. Therefore, the + stream is left exactly where it was left before: + + - [`operator>>`](../operator_gtgt.md) and [`sax_parse`](../basic_json/sax_parse.md) with `strict = false` can still + read several values from one stream, and the stream can still be used with other `std::istream` functions after a + value has been read. + - With [`JSON_PRECISE_STREAM_POSITION`](json_precise_stream_position.md), the stream is still positioned right after + a parsed number. + - Exception messages, including the line, the column, and the last read token, are unchanged. + +!!! note "Performance" + + Parsing from a `std::ifstream` or `std::istringstream` is typically 1.5 to 2 times as fast as before, and reading + many small values from one stream with `operator>>` 1.4 to 4 times as fast. Parsing from a stream is then within + about 10 to 20 % of first reading the whole file into a `std::string` and parsing that. + + A stream buffer without a get area (one that only overrides `underflow()` and `uflow()` and hands out every + character from there) offers nothing to scan in bulk, so it is read one character at a time as before. + +!!! note "Requirements on the stream buffer" + + The bulk scan relies on the following behavior, which every stream buffer of the standard library and every stream + buffer that keeps its input in its get area has: + + - When `sgetc()` returns a character while `gptr() < egptr()`, that character is `*gptr()`; when `sbumpc()` returns + one, it was at `gptr()` before the call. + - Characters in the get area are not changed while they are in the get area. + - `underflow()` or `uflow()` either sets up a get area that holds the returned character at `gptr()` (or at + `gptr() - 1` after `uflow()`), or leaves the get area empty. + + A stream buffer that returns characters from `uflow()` while leaving unrelated characters in its get area, or that + changes the characters in its get area in place, does not meet these requirements. Define the macro to `1` for + such stream buffers. + +!!! warning "Define consistently" + + The macro selects between two definitions of the library's internal stream input adapter. It must therefore be + defined identically for **every** translation unit that includes the library; mixing translation units that define + it with ones that do not is an ODR violation. Prefer setting it as a compile definition on the target rather than + with `#!cpp #define` in individual source files. + +!!! note "Other inputs" + + Strings, string views, vectors, arrays, and pointer or iterator ranges over contiguous memory are scanned in bulk + anyway. Reading from a `#!cpp FILE*` is not affected by this macro. + +## Examples + +??? example "Example: reading values from a stream (default)" + + The stream is scanned in bulk, and the values are read one after the other: + + ```cpp + #include + #include + #include + + using json = nlohmann::json; + + int main() + { + std::istringstream input(R"({"id": 1} {"id": 2} 42 rest)"); + json j1, j2, j3; + input >> j1 >> j2 >> j3; // {"id":1}, {"id":2}, 42 + + std::string word; + input >> word; // "rest" + } + ``` + +??? example "Example: reading one character at a time (macro defined to 1)" + + With the macro, the same program reads the stream one character at a time, with the same results: + + ```cpp + #define JSON_DISABLE_STREAM_BULK_SCAN 1 + #include + #include + #include + + using json = nlohmann::json; + + int main() + { + std::istringstream input(R"({"id": 1} {"id": 2} 42 rest)"); + json j1, j2, j3; + input >> j1 >> j2 >> j3; // {"id":1}, {"id":2}, 42 + + std::string word; + input >> word; // "rest" + } + ``` + +## See also + +- [**parse**](../basic_json/parse.md) - deserialize from a compatible input +- [**operator>>**](../operator_gtgt.md) - deserialize from stream +- [**sax_parse**](../basic_json/sax_parse.md) - generate SAX events +- [**JSON_PRECISE_STREAM_POSITION**](json_precise_stream_position.md) - leave an input stream positioned right after a + parsed number +- [Parsing: input sources and performance](../../features/parsing/index.md#input-sources-and-performance) + +## Version history + +- Added in version 3.13.0. diff --git a/docs/mkdocs/docs/api/macros/json_precise_stream_position.md b/docs/mkdocs/docs/api/macros/json_precise_stream_position.md index b3145fc66..0d0114da3 100644 --- a/docs/mkdocs/docs/api/macros/json_precise_stream_position.md +++ b/docs/mkdocs/docs/api/macros/json_precise_stream_position.md @@ -72,6 +72,12 @@ The default value is `0` (disabled — existing behavior is preserved). distinct symbol names. Translation units compiled with and without it can therefore be linked into the same program without One Definition Rule (ODR) violations, but they cannot exchange instances of library types. +!!! note "Bulk scanning" + + The parser scans the characters in the stream buffer in bulk (see + [`JSON_DISABLE_STREAM_BULK_SCAN`](json_disable_stream_bulk_scan.md)), but it never reads ahead. The stream is left + at the same position with and without bulk scanning, in both modes of this macro. + !!! tip "Workaround without the macro" Separate the values in the stream with whitespace. The character consumed after a number is then the separator, diff --git a/docs/mkdocs/docs/api/macros/json_use_simdutf.md b/docs/mkdocs/docs/api/macros/json_use_simdutf.md index 99dc09892..6e3d0406e 100644 --- a/docs/mkdocs/docs/api/macros/json_use_simdutf.md +++ b/docs/mkdocs/docs/api/macros/json_use_simdutf.md @@ -12,8 +12,9 @@ non-ASCII characters (e.g. CJK or emoji) this can validate several times faster. This is an **opt-in external dependency**. The library itself remains header-only and its behavior is unchanged: the same input is accepted or rejected either way, and every parse error is reported at the same position with the same message (simdutf is only used to fast-path *valid* runs; anything it flags falls back to the scalar path so the exact -diagnostic is preserved). Streaming inputs (files, `std::istream`, wide strings, user-defined adapters) always use the -scalar path. +diagnostic is preserved). A `std::istream` uses simdutf for the characters its stream buffer holds (see +[`JSON_DISABLE_STREAM_BULK_SCAN`](json_disable_stream_bulk_scan.md)); other streaming inputs (`FILE*`, wide strings, +user-defined adapters) always use the scalar path. When `JSON_USE_SIMDUTF` is defined you must make the `simdutf.h` header available on the include path and link the simdutf library. When it is not defined, no simdutf header is included and there is no dependency. diff --git a/docs/mkdocs/docs/api/operator_gtgt.md b/docs/mkdocs/docs/api/operator_gtgt.md index 334d9539e..741183f2f 100644 --- a/docs/mkdocs/docs/api/operator_gtgt.md +++ b/docs/mkdocs/docs/api/operator_gtgt.md @@ -121,6 +121,7 @@ being read. - [`JSON_STRICT_NUL_HANDLING`](macros/json_strict_nul_handling.md) - opt in to rejecting a NUL byte in the input instead of treating it as end of input - [`JSON_PRECISE_STREAM_POSITION`](macros/json_precise_stream_position.md) - opt in to leaving the stream positioned right after a number +- [`JSON_DISABLE_STREAM_BULK_SCAN`](macros/json_disable_stream_bulk_scan.md) - read the stream one character at a time ## Version history @@ -133,3 +134,6 @@ being read. crash (`std::terminate`) when `i` has `eofbit` in its exception mask, in version 3.13.0. - Changed to the strong exception safety guarantee in version 3.13.0: `j` is no longer left with a partially parsed value if parsing throws. +- Scans the characters in the stream buffer of `i` in bulk in version 3.13.0; the position of `i` after the call is + unchanged ([`JSON_DISABLE_STREAM_BULK_SCAN`](macros/json_disable_stream_bulk_scan.md) restores reading one character + at a time). diff --git a/docs/mkdocs/docs/features/macros.md b/docs/mkdocs/docs/features/macros.md index 5df3671e2..396862170 100644 --- a/docs/mkdocs/docs/features/macros.md +++ b/docs/mkdocs/docs/features/macros.md @@ -133,6 +133,14 @@ When defined, default parse and serialize functions for enums are excluded and h See [full documentation of `JSON_DISABLE_ENUM_SERIALIZATION`](../api/macros/json_disable_enum_serialization.md). +## `JSON_DISABLE_STREAM_BULK_SCAN` + +When defined to `1`, parsing from a `#!cpp std::istream` reads one character at a time instead of scanning the +characters that the stream buffer holds in bulk. Results are the same either way; the macro is an escape hatch for +stream buffers that do not follow the usual `std::streambuf` contract. The default value is `0`. + +See [full documentation of `JSON_DISABLE_STREAM_BULK_SCAN`](../api/macros/json_disable_stream_bulk_scan.md). + ## `JSON_DISABLE_TUPLE_REFERENCE_CONVERSION` When defined to `1`, a JSON value can no longer be created from a one-element `std::tuple` holding a reference to a JSON diff --git a/docs/mkdocs/docs/features/parsing/index.md b/docs/mkdocs/docs/features/parsing/index.md index 9e5371d95..3099b7751 100644 --- a/docs/mkdocs/docs/features/parsing/index.md +++ b/docs/mkdocs/docs/features/parsing/index.md @@ -55,6 +55,28 @@ whitespace, because `operator>>` consumes the character that terminates a number [`JSON_PRECISE_STREAM_POSITION`](../../api/macros/json_precise_stream_position.md) is defined to `1` — see the [`operator>>` notes](../../api/operator_gtgt.md#notes) for details and examples. +## Input sources and performance + +All inputs produce the same values and the same exceptions, but they are read in different ways, which makes some of +them faster than others: + +| input | how it is read | +|--------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------| +| strings, string views, string literals, contiguous containers, pointer or iterator ranges over contiguous memory | the whole input is scanned in bulk: runs of string characters, numbers, and indentation are recognized several bytes at a time | +| `#!cpp std::istream` (`#!cpp std::ifstream`, `#!cpp std::istringstream`, ...) | the characters the stream buffer holds are scanned in bulk like a string; the stream buffer is asked for more when they are used up | +| `#!cpp FILE*` | one character at a time | +| other iterator ranges (e.g., `#!cpp std::list`, `#!cpp std::istreambuf_iterator`), wide strings | one character at a time | + +Parsing a file from a `#!cpp std::ifstream` is therefore only about 10 to 20 % slower than reading the whole file into a +`#!cpp std::string` first and parsing that, and it needs no memory for a copy of the file. The stream is never read +ahead: after parsing, it is positioned exactly as if every character had been read one at a time, so +[`operator>>`](../../api/operator_gtgt.md) can still read several values from one stream (see also +[JSON Lines](json_lines.md)). + +The macro [`JSON_DISABLE_STREAM_BULK_SCAN`](../../api/macros/json_disable_stream_bulk_scan.md) switches the bulk +scanning of streams off without changing any result, for stream buffers that do not follow the usual `std::streambuf` +contract. + ## SAX vs. DOM parsing The library offers two parsing models: diff --git a/docs/mkdocs/mkdocs.yml b/docs/mkdocs/mkdocs.yml index 74b6c4d46..be17eeb0e 100644 --- a/docs/mkdocs/mkdocs.yml +++ b/docs/mkdocs/mkdocs.yml @@ -297,6 +297,7 @@ nav: - 'JSON_DELETE_DEPRECATED_FUNCTIONS': api/macros/json_delete_deprecated_functions.md - 'JSON_DIAGNOSTIC_POSITIONS': api/macros/json_diagnostic_positions.md - 'JSON_DISABLE_ENUM_SERIALIZATION': api/macros/json_disable_enum_serialization.md + - 'JSON_DISABLE_STREAM_BULK_SCAN': api/macros/json_disable_stream_bulk_scan.md - 'JSON_DISABLE_TUPLE_REFERENCE_CONVERSION': api/macros/json_disable_tuple_reference_conversion.md - 'JSON_HAS_CPP_11, JSON_HAS_CPP_14, JSON_HAS_CPP_17, JSON_HAS_CPP_20, JSON_HAS_CPP_23, JSON_HAS_CPP_26': api/macros/json_has_cpp_11.md - 'JSON_HAS_EXPERIMENTAL_FILESYSTEM, JSON_HAS_FILESYSTEM': api/macros/json_has_filesystem.md diff --git a/include/nlohmann/detail/input/input_adapters.hpp b/include/nlohmann/detail/input/input_adapters.hpp index 2df78dabb..4102dae3a 100644 --- a/include/nlohmann/detail/input/input_adapters.hpp +++ b/include/nlohmann/detail/input/input_adapters.hpp @@ -14,10 +14,12 @@ #include // uint32_t #include // strlen #include // begin, end, iterator_traits, random_access_iterator_tag, distance, next, reverse_iterator +#include // numeric_limits #include // streambuf #include // string, char_traits #include // enable_if, is_base_of, is_pointer, is_integral, remove_pointer #include // pair, declval +#include // vector #ifndef JSON_NO_IO #include // FILE * @@ -30,6 +32,7 @@ #include #include + NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail { @@ -127,15 +130,26 @@ class input_stream_adapter input_stream_adapter& operator=(input_stream_adapter&) = delete; input_stream_adapter& operator=(input_stream_adapter&&) = delete; -#if JSON_PRECISE_STREAM_POSITION input_stream_adapter(input_stream_adapter&& rhs) noexcept - : is(rhs.is), sb(rhs.sb), lookahead(rhs.lookahead) + : is(rhs.is), sb(rhs.sb) +#if JSON_PRECISE_STREAM_POSITION + , lookahead(rhs.lookahead) +#endif +#if !JSON_DISABLE_STREAM_BULK_SCAN + , token_carry(std::move(rhs.token_carry)) + , token_segment(rhs.token_segment) + , current_in_get_area(rhs.current_in_get_area) + , tracking_token(rhs.tracking_token) +#endif { rhs.is = nullptr; rhs.sb = nullptr; +#if JSON_PRECISE_STREAM_POSITION rhs.lookahead = false; +#endif } +#if JSON_PRECISE_STREAM_POSITION // Whether the character last returned by get_character() can be given back // to the input with release_lookahead(). static constexpr bool supports_lookahead = true; @@ -155,17 +169,41 @@ class input_stream_adapter sb->sbumpc(); } +#if !JSON_DISABLE_STREAM_BULK_SCAN + // sgetc() refills the get area if it is exhausted + const bool refill = streambuf_access::get_ptr(sb) == streambuf_access::get_end(sb); + if (JSON_HEDLEY_UNLIKELY(refill)) + { + save_token_segment(); + } +#endif + auto res = sb->sgetc(); // set eof manually, as we don't use the istream interface. if (JSON_HEDLEY_UNLIKELY(res == std::char_traits::eof())) { // there is nothing to step over next time lookahead = false; +#if !JSON_DISABLE_STREAM_BULK_SCAN + current_in_get_area = false; +#endif is->clear(is->rdstate() | std::ios::eofbit); } else { lookahead = true; +#if !JSON_DISABLE_STREAM_BULK_SCAN + if (JSON_HEDLEY_UNLIKELY(refill)) + { + // the peeked character is at gptr(), unless the streambuf has + // no get area + after_refill(res, streambuf_access::get_ptr(sb) != streambuf_access::get_end(sb)); + } + else + { + current_in_get_area = true; + } +#endif } return res; } @@ -179,13 +217,6 @@ class input_stream_adapter lookahead = false; } #else - input_stream_adapter(input_stream_adapter&& rhs) noexcept - : is(rhs.is), sb(rhs.sb) - { - rhs.is = nullptr; - rhs.sb = nullptr; - } - // std::istream/std::streambuf use std::char_traits::to_int_type, to // ensure that std::char_traits::eof() and the character 0xFF do not // end up as the same value, e.g., 0xFFFFFFFF. @@ -194,12 +225,37 @@ class input_stream_adapter // stays consumed after parsing; see JSON_PRECISE_STREAM_POSITION. std::char_traits::int_type get_character() { +#if !JSON_DISABLE_STREAM_BULK_SCAN + // sbumpc() refills the get area if it is exhausted + const bool refill = streambuf_access::get_ptr(sb) == streambuf_access::get_end(sb); + if (JSON_HEDLEY_UNLIKELY(refill)) + { + save_token_segment(); + } +#endif + auto res = sb->sbumpc(); // set eof manually, as we don't use the istream interface. if (JSON_HEDLEY_UNLIKELY(res == std::char_traits::eof())) { +#if !JSON_DISABLE_STREAM_BULK_SCAN + current_in_get_area = false; +#endif is->clear(is->rdstate() | std::ios::eofbit); } +#if !JSON_DISABLE_STREAM_BULK_SCAN + else if (JSON_HEDLEY_UNLIKELY(refill)) + { + // the consumed character is at gptr() - 1, unless the streambuf + // has no get area and returned it from uflow() + const char* const next = streambuf_access::get_ptr(sb); + after_refill(res, next != nullptr && next != streambuf_access::get_begin(sb)); + } + else + { + current_in_get_area = true; + } +#endif return res; } #endif @@ -218,7 +274,165 @@ class input_stream_adapter return res; } +#if !JSON_DISABLE_STREAM_BULK_SCAN + // The lexer may scan the characters in the get area of the streambuf + // directly (SWAR fast paths). The get area is only a window into the + // input: it ends wherever the streambuf's buffer ends, and is refilled by + // the next get_character(). The lexer therefore keeps counting the + // position itself and treats the end of the window as "unknown", not as + // the end of the input. + static constexpr bool supports_bulk_scan = true; + static constexpr bool bulk_window = true; + + // Pointer to the next unread character; only valid when + // bulk_remaining() > 0. The character before it is the one last returned + // by get_character(). + const char* bulk_data() const + { + return current_end(); + } + + // Number of unread characters in the get area. + std::size_t bulk_remaining() const + { + if (!current_in_get_area) + { + return 0; + } + return static_cast(streambuf_access::get_end(sb) - current_end()); + } + + // Consume @a n characters previously inspected via bulk_data(); the last + // of them becomes the character last returned by get_character(). + void bulk_skip(std::size_t n) + { + JSON_ASSERT(n <= bulk_remaining()); + // gbump() takes an int, but a get area (e.g., of a std::stringbuf) can + // be larger + constexpr auto max_bump = static_cast((std::numeric_limits::max)()); + for (; n > max_bump; n -= max_bump) + { + streambuf_access::bump(sb, static_cast(max_bump)); + } + streambuf_access::bump(sb, static_cast(n)); + } + + // The adapter keeps the current token for the lexer's error messages: the + // part in the get area is only remembered as a pointer, and copied out + // when the get area is about to be refilled. + static constexpr bool supports_token_tracking = true; + + // Start a token with the characters read from now on. Until this is + // called, nothing is tracked (the binary readers never call it). + void begin_token() noexcept + { + tracking_token = true; + token_carry.clear(); + token_segment = current_end(); + } + + // Start a new token with the character @a c last returned by + // get_character(). + void mark_token_start(std::char_traits::int_type c) + { + tracking_token = true; + token_carry.clear(); + if (current_in_get_area) + { + token_segment = current_end() - 1; + } + else + { + token_segment = nullptr; + token_carry.push_back(std::char_traits::to_char_type(c)); + } + } + + // Copy the characters read since mark_token_start() to @a out, without the + // last one if @a drop_last. + void copy_token(std::vector& out, bool drop_last) const + { + out = token_carry; + if (token_segment != nullptr) + { + out.insert(out.end(), token_segment, current_end()); + } + if (drop_last && !out.empty()) + { + out.pop_back(); + } + } +#endif + private: +#if !JSON_DISABLE_STREAM_BULK_SCAN + // Reads the protected get-area pointers of any std::streambuf. Naming the + // members through a derived class makes forming the member pointers + // accessible; calling them on another streambuf object is then ordinary + // member access through a pointer to member. + struct streambuf_access : std::streambuf + { + static const char* get_ptr(std::streambuf* s) + { + return (s->*(&streambuf_access::gptr))(); + } + static const char* get_end(std::streambuf* s) + { + return (s->*(&streambuf_access::egptr))(); + } + static const char* get_begin(std::streambuf* s) + { + return (s->*(&streambuf_access::eback))(); + } + static void bump(std::streambuf* s, int n) + { + (s->*(&streambuf_access::gbump))(n); + } + }; + + // one past the character last returned by get_character(), if that is in + // the get area + const char* current_end() const + { +#if JSON_PRECISE_STREAM_POSITION + // the character is peeked at gptr() and not stepped over yet, unless + // it was given back with release_lookahead() + return streambuf_access::get_ptr(sb) + (lookahead ? 1 : 0); +#else + return streambuf_access::get_ptr(sb); +#endif + } + + // the get area is about to be replaced: copy the token's part in it + void save_token_segment() + { + if (token_segment != nullptr) + { + token_carry.insert(token_carry.end(), token_segment, streambuf_access::get_end(sb)); + token_segment = nullptr; + } + } + + // character @a c was just read into a new get area (@a in_get_area) or + // returned without one + void after_refill(std::char_traits::int_type c, bool in_get_area) + { + current_in_get_area = in_get_area; + if (!tracking_token) + { + return; + } + if (in_get_area) + { + token_segment = current_end() - 1; + } + else + { + token_carry.push_back(std::char_traits::to_char_type(c)); + } + } +#endif + #if JSON_PRECISE_STREAM_POSITION // Step over the character last returned by get_character(). The character // has already been peeked successfully, so for every streambuf with a get @@ -240,6 +454,16 @@ class input_stream_adapter /// whether get_character() peeked a character that is not consumed yet bool lookahead = false; #endif +#if !JSON_DISABLE_STREAM_BULK_SCAN + /// the part of the current token that was in earlier get areas + std::vector token_carry{}; // NOLINT(readability-redundant-member-init) + /// start of the current token's part in the get area, or nullptr + const char* token_segment = nullptr; + /// whether the character last returned by get_character() is in the get area + bool current_in_get_area = false; + /// whether a token is tracked at all; the binary readers never start one + bool tracking_token = false; +#endif }; #endif // JSON_NO_IO diff --git a/include/nlohmann/detail/macro_scope.hpp b/include/nlohmann/detail/macro_scope.hpp index f5f6473b9..83cceaabd 100644 --- a/include/nlohmann/detail/macro_scope.hpp +++ b/include/nlohmann/detail/macro_scope.hpp @@ -895,3 +895,7 @@ #ifndef JSON_DISABLE_TUPLE_REFERENCE_CONVERSION #define JSON_DISABLE_TUPLE_REFERENCE_CONVERSION 0 #endif + +#ifndef JSON_DISABLE_STREAM_BULK_SCAN + #define JSON_DISABLE_STREAM_BULK_SCAN 0 +#endif diff --git a/include/nlohmann/detail/macro_unscope.hpp b/include/nlohmann/detail/macro_unscope.hpp index 494cba0d8..c80e1168f 100644 --- a/include/nlohmann/detail/macro_unscope.hpp +++ b/include/nlohmann/detail/macro_unscope.hpp @@ -21,6 +21,7 @@ #undef JSON_NO_UNIQUE_ADDRESS #undef JSON_DISABLE_ENUM_SERIALIZATION #undef JSON_DISABLE_TUPLE_REFERENCE_CONVERSION +#undef JSON_DISABLE_STREAM_BULK_SCAN #ifndef JSON_TEST_KEEP_MACROS #undef JSON_CATCH diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 1314b5727..e3afae1f8 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -3238,6 +3238,10 @@ JSON_HEDLEY_DIAGNOSTIC_POP #define JSON_DISABLE_TUPLE_REFERENCE_CONVERSION 0 #endif +#ifndef JSON_DISABLE_STREAM_BULK_SCAN + #define JSON_DISABLE_STREAM_BULK_SCAN 0 +#endif + #if JSON_HAS_THREE_WAY_COMPARISON #include // partial_ordering #endif @@ -7974,10 +7978,12 @@ NLOHMANN_JSON_NAMESPACE_END #include // uint32_t #include // strlen #include // begin, end, iterator_traits, random_access_iterator_tag, distance, next, reverse_iterator +#include // numeric_limits #include // streambuf #include // string, char_traits #include // enable_if, is_base_of, is_pointer, is_integral, remove_pointer #include // pair, declval +#include // vector #ifndef JSON_NO_IO #include // FILE * @@ -7995,6 +8001,7 @@ NLOHMANN_JSON_NAMESPACE_END // #include + NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail { @@ -8092,15 +8099,26 @@ class input_stream_adapter input_stream_adapter& operator=(input_stream_adapter&) = delete; input_stream_adapter& operator=(input_stream_adapter&&) = delete; -#if JSON_PRECISE_STREAM_POSITION input_stream_adapter(input_stream_adapter&& rhs) noexcept - : is(rhs.is), sb(rhs.sb), lookahead(rhs.lookahead) + : is(rhs.is), sb(rhs.sb) +#if JSON_PRECISE_STREAM_POSITION + , lookahead(rhs.lookahead) +#endif +#if !JSON_DISABLE_STREAM_BULK_SCAN + , token_carry(std::move(rhs.token_carry)) + , token_segment(rhs.token_segment) + , current_in_get_area(rhs.current_in_get_area) + , tracking_token(rhs.tracking_token) +#endif { rhs.is = nullptr; rhs.sb = nullptr; +#if JSON_PRECISE_STREAM_POSITION rhs.lookahead = false; +#endif } +#if JSON_PRECISE_STREAM_POSITION // Whether the character last returned by get_character() can be given back // to the input with release_lookahead(). static constexpr bool supports_lookahead = true; @@ -8120,17 +8138,41 @@ class input_stream_adapter sb->sbumpc(); } +#if !JSON_DISABLE_STREAM_BULK_SCAN + // sgetc() refills the get area if it is exhausted + const bool refill = streambuf_access::get_ptr(sb) == streambuf_access::get_end(sb); + if (JSON_HEDLEY_UNLIKELY(refill)) + { + save_token_segment(); + } +#endif + auto res = sb->sgetc(); // set eof manually, as we don't use the istream interface. if (JSON_HEDLEY_UNLIKELY(res == std::char_traits::eof())) { // there is nothing to step over next time lookahead = false; +#if !JSON_DISABLE_STREAM_BULK_SCAN + current_in_get_area = false; +#endif is->clear(is->rdstate() | std::ios::eofbit); } else { lookahead = true; +#if !JSON_DISABLE_STREAM_BULK_SCAN + if (JSON_HEDLEY_UNLIKELY(refill)) + { + // the peeked character is at gptr(), unless the streambuf has + // no get area + after_refill(res, streambuf_access::get_ptr(sb) != streambuf_access::get_end(sb)); + } + else + { + current_in_get_area = true; + } +#endif } return res; } @@ -8144,13 +8186,6 @@ class input_stream_adapter lookahead = false; } #else - input_stream_adapter(input_stream_adapter&& rhs) noexcept - : is(rhs.is), sb(rhs.sb) - { - rhs.is = nullptr; - rhs.sb = nullptr; - } - // std::istream/std::streambuf use std::char_traits::to_int_type, to // ensure that std::char_traits::eof() and the character 0xFF do not // end up as the same value, e.g., 0xFFFFFFFF. @@ -8159,12 +8194,37 @@ class input_stream_adapter // stays consumed after parsing; see JSON_PRECISE_STREAM_POSITION. std::char_traits::int_type get_character() { +#if !JSON_DISABLE_STREAM_BULK_SCAN + // sbumpc() refills the get area if it is exhausted + const bool refill = streambuf_access::get_ptr(sb) == streambuf_access::get_end(sb); + if (JSON_HEDLEY_UNLIKELY(refill)) + { + save_token_segment(); + } +#endif + auto res = sb->sbumpc(); // set eof manually, as we don't use the istream interface. if (JSON_HEDLEY_UNLIKELY(res == std::char_traits::eof())) { +#if !JSON_DISABLE_STREAM_BULK_SCAN + current_in_get_area = false; +#endif is->clear(is->rdstate() | std::ios::eofbit); } +#if !JSON_DISABLE_STREAM_BULK_SCAN + else if (JSON_HEDLEY_UNLIKELY(refill)) + { + // the consumed character is at gptr() - 1, unless the streambuf + // has no get area and returned it from uflow() + const char* const next = streambuf_access::get_ptr(sb); + after_refill(res, next != nullptr && next != streambuf_access::get_begin(sb)); + } + else + { + current_in_get_area = true; + } +#endif return res; } #endif @@ -8183,7 +8243,165 @@ class input_stream_adapter return res; } +#if !JSON_DISABLE_STREAM_BULK_SCAN + // The lexer may scan the characters in the get area of the streambuf + // directly (SWAR fast paths). The get area is only a window into the + // input: it ends wherever the streambuf's buffer ends, and is refilled by + // the next get_character(). The lexer therefore keeps counting the + // position itself and treats the end of the window as "unknown", not as + // the end of the input. + static constexpr bool supports_bulk_scan = true; + static constexpr bool bulk_window = true; + + // Pointer to the next unread character; only valid when + // bulk_remaining() > 0. The character before it is the one last returned + // by get_character(). + const char* bulk_data() const + { + return current_end(); + } + + // Number of unread characters in the get area. + std::size_t bulk_remaining() const + { + if (!current_in_get_area) + { + return 0; + } + return static_cast(streambuf_access::get_end(sb) - current_end()); + } + + // Consume @a n characters previously inspected via bulk_data(); the last + // of them becomes the character last returned by get_character(). + void bulk_skip(std::size_t n) + { + JSON_ASSERT(n <= bulk_remaining()); + // gbump() takes an int, but a get area (e.g., of a std::stringbuf) can + // be larger + constexpr auto max_bump = static_cast((std::numeric_limits::max)()); + for (; n > max_bump; n -= max_bump) + { + streambuf_access::bump(sb, static_cast(max_bump)); + } + streambuf_access::bump(sb, static_cast(n)); + } + + // The adapter keeps the current token for the lexer's error messages: the + // part in the get area is only remembered as a pointer, and copied out + // when the get area is about to be refilled. + static constexpr bool supports_token_tracking = true; + + // Start a token with the characters read from now on. Until this is + // called, nothing is tracked (the binary readers never call it). + void begin_token() noexcept + { + tracking_token = true; + token_carry.clear(); + token_segment = current_end(); + } + + // Start a new token with the character @a c last returned by + // get_character(). + void mark_token_start(std::char_traits::int_type c) + { + tracking_token = true; + token_carry.clear(); + if (current_in_get_area) + { + token_segment = current_end() - 1; + } + else + { + token_segment = nullptr; + token_carry.push_back(std::char_traits::to_char_type(c)); + } + } + + // Copy the characters read since mark_token_start() to @a out, without the + // last one if @a drop_last. + void copy_token(std::vector& out, bool drop_last) const + { + out = token_carry; + if (token_segment != nullptr) + { + out.insert(out.end(), token_segment, current_end()); + } + if (drop_last && !out.empty()) + { + out.pop_back(); + } + } +#endif + private: +#if !JSON_DISABLE_STREAM_BULK_SCAN + // Reads the protected get-area pointers of any std::streambuf. Naming the + // members through a derived class makes forming the member pointers + // accessible; calling them on another streambuf object is then ordinary + // member access through a pointer to member. + struct streambuf_access : std::streambuf + { + static const char* get_ptr(std::streambuf* s) + { + return (s->*(&streambuf_access::gptr))(); + } + static const char* get_end(std::streambuf* s) + { + return (s->*(&streambuf_access::egptr))(); + } + static const char* get_begin(std::streambuf* s) + { + return (s->*(&streambuf_access::eback))(); + } + static void bump(std::streambuf* s, int n) + { + (s->*(&streambuf_access::gbump))(n); + } + }; + + // one past the character last returned by get_character(), if that is in + // the get area + const char* current_end() const + { +#if JSON_PRECISE_STREAM_POSITION + // the character is peeked at gptr() and not stepped over yet, unless + // it was given back with release_lookahead() + return streambuf_access::get_ptr(sb) + (lookahead ? 1 : 0); +#else + return streambuf_access::get_ptr(sb); +#endif + } + + // the get area is about to be replaced: copy the token's part in it + void save_token_segment() + { + if (token_segment != nullptr) + { + token_carry.insert(token_carry.end(), token_segment, streambuf_access::get_end(sb)); + token_segment = nullptr; + } + } + + // character @a c was just read into a new get area (@a in_get_area) or + // returned without one + void after_refill(std::char_traits::int_type c, bool in_get_area) + { + current_in_get_area = in_get_area; + if (!tracking_token) + { + return; + } + if (in_get_area) + { + token_segment = current_end() - 1; + } + else + { + token_carry.push_back(std::char_traits::to_char_type(c)); + } + } +#endif + #if JSON_PRECISE_STREAM_POSITION // Step over the character last returned by get_character(). The character // has already been peeked successfully, so for every streambuf with a get @@ -8205,6 +8423,16 @@ class input_stream_adapter /// whether get_character() peeked a character that is not consumed yet bool lookahead = false; #endif +#if !JSON_DISABLE_STREAM_BULK_SCAN + /// the part of the current token that was in earlier get areas + std::vector token_carry{}; // NOLINT(readability-redundant-member-init) + /// start of the current token's part in the get area, or nullptr + const char* token_segment = nullptr; + /// whether the character last returned by get_character() is in the get area + bool current_in_get_area = false; + /// whether a token is tracked at all; the binary readers never start one + bool tracking_token = false; +#endif }; #endif // JSON_NO_IO @@ -36247,6 +36475,7 @@ struct formatter // NOLINT(cert-dcl58-c #undef JSON_NO_UNIQUE_ADDRESS #undef JSON_DISABLE_ENUM_SERIALIZATION #undef JSON_DISABLE_TUPLE_REFERENCE_CONVERSION +#undef JSON_DISABLE_STREAM_BULK_SCAN #ifndef JSON_TEST_KEEP_MACROS #undef JSON_CATCH diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 9318838bb..a314bfbda 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -369,6 +369,24 @@ json_test_add_test_for(src/unit-diagnostic-positions.cpp MAIN test_main CXX_STANDARDS ${test_cxx_standards} ${test_force} ) +# test reading from a std::istream again without scanning the get area in bulk +json_test_set_test_options(test-stream-bulk-scan_disabled + COMPILE_DEFINITIONS JSON_DISABLE_STREAM_BULK_SCAN=1 +) +json_test_add_test_for(src/unit-stream-bulk-scan.cpp + NAME test-stream-bulk-scan_disabled + MAIN test_main CXX_STANDARDS ${test_cxx_standards} ${test_force} +) + +# test reading from a std::istream again with JSON_PRECISE_STREAM_POSITION +json_test_set_test_options(test-stream-bulk-scan_precise + COMPILE_DEFINITIONS JSON_PRECISE_STREAM_POSITION=1 +) +json_test_add_test_for(src/unit-stream-bulk-scan.cpp + NAME test-stream-bulk-scan_precise + MAIN test_main CXX_STANDARDS ${test_cxx_standards} ${test_force} +) + endif() # *DO NOT* use json_test_set_test_options() below this line diff --git a/tests/src/unit-precise-stream-position.cpp b/tests/src/unit-precise-stream-position.cpp index e2d9bd0c5..0e8a6094f 100644 --- a/tests/src/unit-precise-stream-position.cpp +++ b/tests/src/unit-precise-stream-position.cpp @@ -20,6 +20,7 @@ #include using nlohmann::json; +#include #include #include #include @@ -72,6 +73,40 @@ class no_putback_streambuf : public std::streambuf std::size_t m_pos = 0; }; +// A streambuf that serves its input in get areas of a fixed size, so that the +// end of a number falls on every possible offset of a get area. The lexer scans +// the get area directly; the stream position must not depend on its size. +class chunked_streambuf : public std::streambuf +{ + public: + chunked_streambuf(std::string s, std::size_t chunk_size) + : m_data(std::move(s)), m_chunk_size(chunk_size) + {} + + protected: + int_type underflow() override + { + if (gptr() < egptr()) + { + return traits_type::to_int_type(*gptr()); + } + m_pos += static_cast(egptr() - eback()); + if (m_pos >= m_data.size()) + { + setg(nullptr, nullptr, nullptr); + return traits_type::eof(); + } + char* const begin = &m_data[m_pos]; + setg(begin, begin, begin + (std::min)(m_chunk_size, m_data.size() - m_pos)); + return traits_type::to_int_type(*gptr()); + } + + private: + std::string m_data; + std::size_t m_pos = 0; + std::size_t m_chunk_size; +}; + // read the characters that are left in a stream std::string remaining(std::istream& is) { @@ -263,4 +298,26 @@ TEST_CASE("JSON_PRECISE_STREAM_POSITION") json _; CHECK_THROWS_WITH_AS(_ = json::parse(is), "[json.exception.parse_error.101] parse error: attempting to parse an empty input; check that your input string or stream contains the expected JSON", json::parse_error&); } + + SECTION("the position does not depend on the get area of the streambuf") + { + const std::vector chunk_sizes = {1, 2, 3, 4, 5, 7, 8, 13, 4096}; + for (const std::size_t chunk_size : chunk_sizes) + { + CAPTURE(chunk_size) + chunked_streambuf buf("12345678901234567890true\"a long string value\"-1.25e+3,[1,2]42", chunk_size); + std::istream input(&buf); + json j; + + input >> j; + CHECK(j == 12345678901234567890u); + input >> j; + CHECK(j == true); + input >> j; + CHECK(j == "a long string value"); + input >> j; + CHECK(j == -1250.0); + CHECK(remaining(input) == ",[1,2]42"); + } + } } diff --git a/tests/src/unit-stream-bulk-scan.cpp b/tests/src/unit-stream-bulk-scan.cpp new file mode 100644 index 000000000..f46c4351f --- /dev/null +++ b/tests/src/unit-stream-bulk-scan.cpp @@ -0,0 +1,417 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#include "doctest_compatibility.h" + +// the library undefines its configuration macros at the end of the header +#if defined(JSON_DISABLE_STREAM_BULK_SCAN) && JSON_DISABLE_STREAM_BULK_SCAN + #define TEST_STREAM_BULK_SCAN 0 +#else + #define TEST_STREAM_BULK_SCAN 1 +#endif + +#include +using nlohmann::json; + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// The lexer scans the characters in the get area of a std::istream's +// streambuf directly instead of reading them one at a time. These tests pin +// that this is not observable: for any get area size - including a streambuf +// without a get area, which still takes the character-at-a-time path - the +// parsed values, the exception messages (including the "last read" token), +// the stream state, and the characters left in the stream are the same. See +// unit-stream-bulk-scan-disabled.cpp for JSON_DISABLE_STREAM_BULK_SCAN. + +namespace +{ +// A streambuf that serves its input in get areas of a fixed size, so that +// tokens, numbers, UTF-8 sequences, and indentation cross the end of the get +// area at every possible offset. With a size of 0, it keeps no get area at all +// and serves every character from underflow()/uflow(). +class chunked_streambuf : public std::streambuf +{ + public: + chunked_streambuf(std::string s, std::size_t chunk_size) + : m_data(std::move(s)), m_chunk_size(chunk_size) + {} + + protected: + int_type underflow() override + { + if (m_chunk_size == 0) + { + return m_pos < m_data.size() ? traits_type::to_int_type(m_data[m_pos]) : traits_type::eof(); + } + if (gptr() < egptr()) + { + return traits_type::to_int_type(*gptr()); + } + // the previous get area has been consumed + m_pos += static_cast(egptr() - eback()); + if (m_pos >= m_data.size()) + { + setg(nullptr, nullptr, nullptr); + return traits_type::eof(); + } + char* const begin = &m_data[m_pos]; + setg(begin, begin, begin + (std::min)(m_chunk_size, m_data.size() - m_pos)); + return traits_type::to_int_type(*gptr()); + } + + int_type uflow() override + { + if (m_chunk_size == 0) + { + return m_pos < m_data.size() ? traits_type::to_int_type(m_data[m_pos++]) : traits_type::eof(); + } + return std::streambuf::uflow(); + } + + private: + std::string m_data; + std::size_t m_pos = 0; + std::size_t m_chunk_size; +}; + +// how a test reads from the stream +enum class read_mode +{ + extract, // up to four times operator>>, as for concatenated values + parse, // json::parse(stream) + parse_lenient, // json::parse(stream) with comments +}; + +// Read @a input from a chunked_streambuf and describe everything observable: +// the values or the exception, the stream state after each value, and the +// characters left in the stream. +std::string observe(const std::string& input, std::size_t chunk_size, read_mode mode) +{ + chunked_streambuf buf(input, chunk_size); + std::istream is(&buf); + std::string result; + for (int i = 0; i < 4; ++i) + { + json j; + try + { + if (mode == read_mode::extract) + { + is >> j; + } + else + { + j = json::parse(is, nullptr, true, mode == read_mode::parse_lenient); + } + result += "value " + j.dump() + "; state " + std::to_string(static_cast(is.rdstate())) + "\n"; + } + catch (const json::exception& e) + { + result += std::string("exception ") + e.what() + "\n"; + break; + } + if (mode != read_mode::extract) + { + break; + } + is.clear(); + } + + is.clear(); + result += "remaining "; + char c = 0; + while (is.get(c)) + { + result += c; + } + return result; +} + +// get area sizes that put every boundary everywhere in short inputs, plus one +// that holds most inputs completely +std::vector chunk_sizes() +{ + return {1, 2, 3, 4, 5, 7, 8, 9, 13, 16, 64, 4096}; +} + +// valid and invalid inputs covering every token type and every bulk path: +// strings (plain runs, escapes, UTF-8), numbers (all grammar parts, too large +// for an integer), and indentation +std::vector corpus() +{ + return + { + // numbers, terminated in every way + "0", "-0", "123", "-123", "1.5", "-1.5e+10", "2E-3", "123456789012345678901234567890", + "18446744073709551615", "18446744073709551616", "-9223372036854775809", + "123 456", "123,456", "123]", "1true", "42\nfoo", "1.25e3}", "7\t8\r9", + // malformed numbers + "1.", "-", "1e", "1e+", "01", "-a", "1.x", "1ee5", + // strings + R"("")", R"("abc")", R"("a long string with a run of ordinary characters to scan in bulk")", + R"("escapes \" \\ \/ \b \f \n \r \t é 😀")", + "\"UTF-8: \xc3\xa9 \xe2\x82\xac \xf0\x9f\x98\x80 done\"", + // malformed strings + R"("unterminated)", "\"control\x01character\"", "\"ill-formed \xc3\x28 UTF-8\"", R"("bad \x escape")", + R"("\ud800 lone surrogate")", + // literals + "true", "false", "null", "tru", "nul", "falsey", + // structures + "[]", "{}", "[1,2,3]", R"({"a":1,"b":[true,false,null],"c":{"d":"e"}})", "[1,]", R"({"a" 1})", "[[[[[]]]]]", + // pretty-printed + "{\n \"key\": [\n 1,\n -2.5,\n \"three\"\n ]\n}", + "[\n \"deep\"\n ]", + "{\n \"a\": 1,\n \"b\": 2x\n}", + // concatenated values and whitespace + "1 2 3", R"("a""b")", "[1][2]", "{}{}", " \n\t 42 \n\t ", "", + // comments (only accepted by parse_lenient) + "/* comment */ 1", "// comment\n[1, /* c */ 2]", + // byte order mark + "\xef\xbb\xbf[1]", + }; +} +} // namespace + +TEST_CASE("parsing from a std::istream scans the get area in bulk") +{ + SECTION("results do not depend on the get area") + { + const std::vector read_modes = {read_mode::extract, read_mode::parse, read_mode::parse_lenient}; + for (const auto& input : corpus()) + { + for (const read_mode mode : read_modes) + { + // no get area: the character-at-a-time path + const std::string expected = observe(input, 0, mode); + for (const std::size_t chunk_size : chunk_sizes()) + { + CAPTURE(input) + CAPTURE(chunk_size) + CAPTURE(static_cast(mode)) + CHECK(observe(input, chunk_size, mode) == expected); + } + } + } + } + + SECTION("values are the same as when parsing from a string") + { + const std::string input = R"({"name": "nlohmann::json", "numbers": [1, -2, 3.5, 4e10, 18446744073709551615],)" + "\n \"text\": \"caf\xc3\xa9 \\u00e9 \\\"quoted\\\"\", \"nested\": [[{}], [[]]], \"flags\": [true, false, null]}"; + const json expected = json::parse(input); + for (const std::size_t chunk_size : chunk_sizes()) + { + CAPTURE(chunk_size) + chunked_streambuf buf(input, chunk_size); + std::istream is(&buf); + CHECK(json::parse(is) == expected); + } + + std::istringstream ss(input); + CHECK(json::parse(ss) == expected); + } + + SECTION("a token that spans several get areas is reported completely") + { + // the error is found after the string has crossed many get areas; the + // token string holds everything since the start of the last token + const std::string input = R"(["this string is longer than every get area", tru])"; + for (const std::size_t chunk_size : chunk_sizes()) + { + CAPTURE(chunk_size) + chunked_streambuf buf(input, chunk_size); + std::istream is(&buf); + json j; + CHECK_THROWS_WITH_AS(is >> j, + "[json.exception.parse_error.101] parse error at line 1, column 50: syntax error while parsing value - invalid literal; last read: '\"this string is longer than every get area\", tru]'", + json::parse_error&); + } + + const std::string unterminated = R"("this string is longer than every get area)"; + for (const std::size_t chunk_size : chunk_sizes()) + { + CAPTURE(chunk_size) + chunked_streambuf buf(unterminated, chunk_size); + std::istream is(&buf); + json j; + CHECK_THROWS_WITH_AS(is >> j, + "[json.exception.parse_error.101] parse error at line 1, column 43: syntax error while parsing value - invalid string: missing closing quote; last read: '\"this string is longer than every get area'", + json::parse_error&); + } + } + + SECTION("an error before the first token reports everything read") + { + chunked_streambuf buf(" \n x", 2); + std::istream is(&buf); + json j; + CHECK_THROWS_WITH_AS(is >> j, + "[json.exception.parse_error.101] parse error at line 2, column 3: syntax error while parsing value - invalid literal; last read: ' x'", + json::parse_error&); + } + + SECTION("lines and columns are counted across get areas") + { + const std::string input = "[\n 1,\n \"two\",\n 3.0,\n x\n]"; + for (const std::size_t chunk_size : chunk_sizes()) + { + CAPTURE(chunk_size) + chunked_streambuf buf(input, chunk_size); + std::istream is(&buf); + json j; + CHECK_THROWS_WITH_AS(is >> j, + "[json.exception.parse_error.101] parse error at line 5, column 5: syntax error while parsing value - invalid literal; last read: '3.0, x'", + json::parse_error&); + } + } + + SECTION("many values from one stream") + { + std::string input; + for (int i = 0; i < 200; ++i) + { + input += R"({"id":)" + std::to_string(i) + R"(,"name":"user)" + std::to_string(i) + R"(","score":)" + std::to_string(i) + ".5}\n"; + input += std::to_string(i * 7919) + "\n"; + } + + for (const std::size_t chunk_size : chunk_sizes()) + { + CAPTURE(chunk_size) + chunked_streambuf buf(input, chunk_size); + std::istream is(&buf); + json j; + int count = 0; + while (!(is >> std::ws).eof()) + { + is >> j; + if (count % 2 == 0) + { + CHECK(j["id"] == count / 2); + } + else + { + CHECK(j == (count / 2) * 7919); + } + ++count; + } + CHECK(count == 400); + } + } + + SECTION("the stream can be used after parsing") + { + for (const std::size_t chunk_size : chunk_sizes()) + { + CAPTURE(chunk_size) + chunked_streambuf buf(R"({"a": "some text"} trailing words 42)", chunk_size); + std::istream is(&buf); + json j; + is >> j; + CHECK(j == json({{"a", "some text"}})); + + std::string word; + int number = 0; + is >> word; + CHECK(word == "trailing"); + is >> word >> number; + CHECK(word == "words"); + CHECK(number == 42); + } + } + + SECTION("std::stringstream and std::ifstream") + { + const std::string input = R"([1, 2.5, "three", {"four": [true, null]}] 5)"; + + std::stringstream ss(input); + json j1; + json j2; + ss >> j1 >> j2; + CHECK(j1 == json::parse(R"([1, 2.5, "three", {"four": [true, null]}])")); + CHECK(j2 == 5); + CHECK(ss.eof()); + + // a file name of its own for every run: the test runs for several C++ + // standards and configurations at the same time, in the same directory + std::random_device rd; + const std::string temp_name = "unit-stream-bulk-scan-" + std::to_string(rd()) + ".tmp"; + const char* const filename = temp_name.c_str(); + { + std::ofstream file(filename); + file << input; + } + { + std::ifstream file(filename); + json k1; + json k2; + file >> k1 >> k2; + CHECK(k1 == j1); + CHECK(k2 == j2); + } + static_cast(std::remove(filename)); + } + + SECTION("binary formats read from a stream") + { + const json j = {{"string", std::string(300, 'x')}, {"array", {1, 2, 3}}, {"number", 1.5}}; + const std::vector cbor = json::to_cbor(j); + const std::string bytes(cbor.begin(), cbor.end()); + for (const std::size_t chunk_size : chunk_sizes()) + { + CAPTURE(chunk_size) + chunked_streambuf buf(bytes, chunk_size); + std::istream is(&buf); + CHECK(json::from_cbor(is) == j); + } + } + +#if TEST_STREAM_BULK_SCAN + SECTION("the adapter keeps no token unless the lexer starts one") + { + // the binary readers read through the same adapter, but never start a + // token; refilling the get area must then not collect the input + chunked_streambuf buf(std::string(1000, 'x'), 7); + std::istream is(&buf); + nlohmann::detail::input_stream_adapter ia(is); + for (int i = 0; i < 1000; ++i) + { + CHECK(ia.get_character() == 'x'); + } + + std::vector token; + ia.copy_token(token, false); + CHECK(token.empty()); + } +#endif + + SECTION("SAX parsing from a stream") + { + const std::string input = R"({"a": [1, "two", 3.5]} rest)"; + for (const std::size_t chunk_size : chunk_sizes()) + { + CAPTURE(chunk_size) + chunked_streambuf buf(input, chunk_size); + std::istream is(&buf); + json j; + nlohmann::detail::json_sax_dom_parser sax(j); + CHECK(json::sax_parse(is, &sax, json::input_format_t::json, false)); + CHECK(j == json::parse(R"({"a": [1, "two", 3.5]})")); + } + } +}