diff --git a/README.md b/README.md index cc3546686..5f651efa9 100644 --- a/README.md +++ b/README.md @@ -496,7 +496,7 @@ bool key(string_t& val); bool parse_error(std::size_t position, const std::string& last_token, const detail::exception& ex); ``` -The return value of each function determines whether parsing should proceed. +The return value of each function determines whether parsing should proceed. For `parse_error`, returning `true` [recovers from the error](https://json.nlohmann.me/features/parsing/error_recovery/): the parser repairs the input and continues. To implement your own SAX handler, proceed as follows: @@ -504,7 +504,7 @@ To implement your own SAX handler, proceed as follows: 2. Create an object of your SAX interface class, e.g. `my_sax`. 3. Call `bool json::sax_parse(input, &my_sax)`; where the first parameter can be any input like a string or an input stream and the second parameter is a pointer to your SAX interface. -Note the `sax_parse` function only returns a `bool` indicating the result of the last executed SAX event. It does not return a `json` value - it is up to you to decide what to do with the SAX events. Furthermore, no exceptions are thrown in case of a parse error -- it is up to you what to do with the exception object passed to your `parse_error` implementation. Internally, the SAX interface is used for the DOM parser (class `json_sax_dom_parser`) as well as the acceptor (`json_sax_acceptor`), see file [`json_sax.hpp`](https://github.com/nlohmann/json/blob/develop/include/nlohmann/detail/input/json_sax.hpp). +Note the `sax_parse` function only returns a `bool` indicating whether the input was parsed without errors and no SAX event returned `false`. It does not return a `json` value - it is up to you to decide what to do with the SAX events. Furthermore, no exceptions are thrown in case of a parse error -- it is up to you what to do with the exception object passed to your `parse_error` implementation. Internally, the SAX interface is used for the DOM parser (class `json_sax_dom_parser`) as well as the acceptor (`json_sax_acceptor`), see file [`json_sax.hpp`](https://github.com/nlohmann/json/blob/develop/include/nlohmann/detail/input/json_sax.hpp). ### STL-like access diff --git a/docs/mkdocs/docs/api/basic_json/sax_parse.md b/docs/mkdocs/docs/api/basic_json/sax_parse.md index d6ce688a7..af5191394 100644 --- a/docs/mkdocs/docs/api/basic_json/sax_parse.md +++ b/docs/mkdocs/docs/api/basic_json/sax_parse.md @@ -90,7 +90,9 @@ The SAX event lister must follow the interface of [`json_sax`](../json_sax/index ## Return value -return value of the last processed SAX event +`#!cpp true` if the input was parsed without errors and no SAX event returned `#!cpp false`; `#!cpp false` otherwise. +In particular, the result is `#!cpp false` for input with errors, even if the SAX parser recovered from all of them +(see [error recovery](../../features/parsing/error_recovery.md)). ## Exception safety @@ -138,6 +140,7 @@ A UTF-8 byte order mark is silently ignored. - Ignoring comments via `ignore_comments` added in version 3.9.0. - Added `ignore_trailing_commas` in version 3.13.0. - Extended container support (1) to include types with lvalue-only ADL `begin`/`end` (matching `std::begin`/`std::end` semantics) in version 3.13.0. +- Recovering from parse errors (see [`parse_error`](../json_sax/parse_error.md)) added in version 3.13.0. - Extended overload (2) to accept heterogeneous iterator+sentinel pairs (C++20 ranges support) in version 3.13.0. - `JSON_PRECISE_STREAM_POSITION` added in version 3.13.0 to optionally leave a `#!cpp std::istream` positioned right after the parsed value when `strict` is `#!cpp false`. diff --git a/docs/mkdocs/docs/api/json_sax/index.md b/docs/mkdocs/docs/api/json_sax/index.md index d66b7a254..2d0af62de 100644 --- a/docs/mkdocs/docs/api/json_sax/index.md +++ b/docs/mkdocs/docs/api/json_sax/index.md @@ -7,7 +7,8 @@ struct json_sax; This class describes the SAX interface used by [sax_parse](../basic_json/sax_parse.md). Each function is called in different situations while the input is parsed. The boolean return value informs the parser whether to continue -processing the input. +processing the input; for [`parse_error`](parse_error.md), it decides whether to +[recover from the error](../../features/parsing/error_recovery.md). ## Template parameters diff --git a/docs/mkdocs/docs/api/json_sax/parse_error.md b/docs/mkdocs/docs/api/json_sax/parse_error.md index e41cb67ff..b2831234a 100644 --- a/docs/mkdocs/docs/api/json_sax/parse_error.md +++ b/docs/mkdocs/docs/api/json_sax/parse_error.md @@ -21,7 +21,14 @@ A parse error occurred. ## Return value -Whether parsing should proceed (**must return `#!cpp false`**). +Whether to recover from the error: + +- `#!cpp false` stops parsing. +- `#!cpp true` recovers from the error: JSON text is repaired and parsing continues; for the binary formats, the value + read so far is completed and parsing stops. See [error recovery](../../features/parsing/error_recovery.md) for how + errors are repaired. + +Either way, [`sax_parse`](../basic_json/sax_parse.md) returns `#!cpp false`. ## Examples @@ -39,6 +46,22 @@ Whether parsing should proceed (**must return `#!cpp false`**). --8<-- "examples/sax_parse.output" ``` +??? example + + The example below shows how a SAX parser recovers from errors. + + ```cpp + --8<-- "examples/sax_parse__error_recovery.cpp" + ``` + + Output: + + ``` + --8<-- "examples/sax_parse__error_recovery.output" + ``` + ## Version history - Added in version 3.2.0. +- Returning `#!cpp true` recovers from the error since version 3.13.0; before, parsing stopped, but the result of + [`sax_parse`](../basic_json/sax_parse.md) could be wrong. diff --git a/docs/mkdocs/docs/examples/sax_parse__error_recovery.cpp b/docs/mkdocs/docs/examples/sax_parse__error_recovery.cpp new file mode 100644 index 000000000..b0fbc470f --- /dev/null +++ b/docs/mkdocs/docs/examples/sax_parse__error_recovery.cpp @@ -0,0 +1,43 @@ +#include +#include +#include + +using json = nlohmann::json; + +// a SAX parser that creates a JSON value like json::parse does, but that +// recovers from parse errors instead of stopping at the first one +class recovering_parser : public nlohmann::detail::json_sax_dom_parser +{ + public: + explicit recovering_parser(json& result) + : nlohmann::detail::json_sax_dom_parser(result, false) + {} + + bool parse_error(std::size_t position, + const std::string& /*last_token*/, + const json::exception& ex) + { + std::cout << "byte " << position << ": " << ex.what() << '\n'; + + // repair the input and continue + return true; + } +}; + +int main() +{ + // JSON text with several mistakes that ends too early + const std::string text = R"({ + "name": "Hello World", + "tags": ["a" "b",], + "valid": tru, + "size": 1., + "nested": {"x": 1)"; + + json result; + recovering_parser sax(result); + const bool valid = json::sax_parse(text, &sax); + + std::cout << "\nvalid JSON: " << std::boolalpha << valid << '\n' + << std::setw(4) << result << std::endl; +} diff --git a/docs/mkdocs/docs/examples/sax_parse__error_recovery.output b/docs/mkdocs/docs/examples/sax_parse__error_recovery.output new file mode 100644 index 000000000..b3c03ec71 --- /dev/null +++ b/docs/mkdocs/docs/examples/sax_parse__error_recovery.output @@ -0,0 +1,19 @@ +byte 49: [json.exception.parse_error.101] parse error at line 3, column 20: syntax error while parsing array - unexpected string literal; expected ']' +byte 51: [json.exception.parse_error.101] parse error at line 3, column 22: syntax error while parsing value - unexpected ']'; expected '[', '{', or a literal +byte 70: [json.exception.parse_error.101] parse error at line 4, column 17: syntax error while parsing value - invalid literal; last read: '"valid": tru,' +byte 86: [json.exception.parse_error.101] parse error at line 5, column 15: syntax error while parsing value - invalid number; expected digit after '.'; last read: '1.,' +byte 109: [json.exception.parse_error.101] parse error at line 6, column 22: syntax error while parsing object - unexpected end of input; expected '}' + +valid JSON: false +{ + "name": "Hello World", + "nested": { + "x": 1 + }, + "size": 1, + "tags": [ + "a", + "b" + ], + "valid": null +} diff --git a/docs/mkdocs/docs/features/parsing/error_recovery.md b/docs/mkdocs/docs/features/parsing/error_recovery.md new file mode 100644 index 000000000..7a2387c72 --- /dev/null +++ b/docs/mkdocs/docs/features/parsing/error_recovery.md @@ -0,0 +1,94 @@ +# Error Recovery + +By default, parsing stops at the first error. With the [SAX interface](sax_interface.md), you can instead ask the +parser to *recover*: to repair the error and continue, so that you get as much as possible out of malformed input, for +instance a file that was cut off, JSON edited by hand, or the output of a language model. + +## Recovering from errors + +The SAX parser's [`parse_error`](../../api/json_sax/parse_error.md) function is called for every error. Its return value +decides what happens next: + +- `#!cpp false` stops parsing. This is what the SAX parsers of the library do, so [`parse`](../../api/basic_json/parse.md) + and [`accept`](../../api/basic_json/accept.md) never recover. +- `#!cpp true` repairs the error and continues parsing. + +When recovering, the SAX parser still receives well-formed events: every `start_object` or `start_array` is followed by +the matching `end_object` or `end_array`, and every `key` is followed by exactly one value. A SAX parser that creates a +JSON value, such as the one in the example below, therefore gets a complete value. Parsing always ends, and +[`sax_parse`](../../api/basic_json/sax_parse.md) returns `#!cpp false` for input that is not valid JSON, even if every +error was repaired. Each token is reported at most once, and the SAX parser can stop at any error by returning +`#!cpp false`. + +!!! example + + The example below derives a SAX parser from the library's parser for `json` values (`json_sax_dom_parser`), + and recovers from all errors. + + ```cpp + --8<-- "examples/sax_parse__error_recovery.cpp" + ``` + + Output: + + ``` + --8<-- "examples/sax_parse__error_recovery.output" + ``` + +## How errors are repaired + +Each error is repaired with the smallest local edit: a missing separator is inserted, a stray token is removed, what can +be read of a broken string or number is kept, and a value that cannot be read at all becomes `#!json null`. + +| Mistake | Repair | Example | Result | +|---------------------------|--------------------------------------------------------------------------------|------------------------------------------|----------------------------| +| missing `,` or `:` | inserted | `#!json [1 2]`, `#!json {"a" 1}` | `[1,2]`, `{"a":1}` | +| missing value | `#!json null` for an object key or between commas in an array | `#!json {"a":}`, `#!json [1,,2]` | `{"a":null}`, `[1,null,2]` | +| trailing comma | removed | `#!json [1,2,]` | `[1,2]` | +| broken string | invalid escapes and bytes are replaced (see below); a line break ends the string | `#!json ["a\qb"]` | `["aqb"]` | +| broken number | the longest valid beginning is kept | `#!json [1., 2e+]` | `[1,2]` | +| unreadable value | `#!json null` | `#!json [1, NaN, tru]` | `[1,null,null]` | +| number too large | passed as infinity, together with its text | `#!json [1e999]` | infinity (see below) | +| stray `:` | removed | `#!json ["a":1]` | `["a",1]` | +| member without a key | skipped up to the next `,` or `}` | `#!json {1:2, "b":3}` | `{"b":3}` | +| wrong closing bracket | closes the innermost array or object | `#!json {"a":[1,2}, "b":3}` | `{"a":[1,2],"b":3}` | +| input ends too early | all open arrays and objects are closed | `#!json {"a":[1,2` | `{"a":[1,2]}` | +| text before the value | skipped | `#!json )]}'{"a":1}` | `{"a":1}` | + +In a string, an unknown escape like `\q` stands for the escaped character (`q`), as in JavaScript. An invalid `\u` +escape, a lone surrogate, and ill-formed UTF-8 are each replaced by U+FFFD (REPLACEMENT CHARACTER), and control +characters are kept. A string without its closing quote ends at the next line break or at the end of the input. + +The input after the top-level value is not repaired: as without recovery, it is reported as an error, and parsing stops. + +## Binary formats + +The binary formats ([BJData](../binary_formats/bjdata.md), [BON8](../binary_formats/bon8.md), +[BSON](../binary_formats/bson.md), [CBOR](../binary_formats/cbor.md), [MessagePack](../binary_formats/messagepack.md), +and [UBJSON](../binary_formats/ubjson.md)) cannot be repaired: a value's size is stored before its content, and every +byte is a valid type marker, so after an error there is no way to tell where the next value begins. Parsing therefore +always stops at the first error. If `parse_error` returns `#!cpp true`, the value read so far is completed before +parsing stops: a key that waits for its value gets `#!json null`, and all open arrays and objects are closed. This keeps +everything before the error of an input that was cut off. + +## Limitations + +- A repair is a guess. For example, `#!json {"a" "b": 1}` could be meant as `#!json {"a": "b"}` or as + `#!json {"a": null, "b": 1}`; it is repaired to the former. Treat recovered values as a best effort, and check the + reported errors. +- A closing bracket always closes the innermost array or object. If a bracket is missing rather than wrong, the + repair differs from the intention: `#!json {"a": {"b": [1, 2}, "c": 3}` is repaired to + `#!json {"a": {"b": [1, 2], "c": 3}}`, although `#!json {"a": {"b": [1, 2]}, "c": 3}` may have been meant. +- Keys without quotes, and strings in single quotes, are not supported; such members are skipped. +- A number that is too large for `number_float_t` is passed as positive or negative infinity. The SAX parser's + `number_float` also gets the number's text, but a JSON value cannot store it, and + [`dump`](../../api/basic_json/dump.md) serializes infinity as `#!json null`. +- When parsing is not strict (see [`sax_parse`](../../api/basic_json/sax_parse.md)), a repair may read parts of the + input after the value, for instance of the next value in a stream of concatenated values. + +## See also + +- [SAX interface](sax_interface.md) - implement a custom SAX handler +- [`parse_error`](../../api/json_sax/parse_error.md) - the SAX event for parse errors +- [`sax_parse`](../../api/basic_json/sax_parse.md) - generate SAX events +- [parsing and exceptions](parse_exceptions.md) - control error handling diff --git a/docs/mkdocs/docs/features/parsing/index.md b/docs/mkdocs/docs/features/parsing/index.md index 476f024fa..fbb04b5f9 100644 --- a/docs/mkdocs/docs/features/parsing/index.md +++ b/docs/mkdocs/docs/features/parsing/index.md @@ -65,7 +65,7 @@ You can influence a DOM parse without switching to the SAX interface by passing When the input is not valid JSON, the `parse` function throws an exception by default. If exceptions are undesired or unavailable, the parser can instead return a discarded value, or [`accept`](../../api/basic_json/accept.md) can be used to only check whether an input is valid JSON. See [parsing and exceptions](parse_exceptions.md) for the available -options. +options. To get as much as possible out of malformed input, a SAX parser can [recover from errors](error_recovery.md). ## See also @@ -76,3 +76,4 @@ options. - [parser callbacks](parser_callbacks.md) - influence the parsing by a callback function - [SAX interface](sax_interface.md) - implement a custom SAX handler - [parsing and exceptions](parse_exceptions.md) - control error handling +- [error recovery](error_recovery.md) - get as much as possible out of malformed input diff --git a/docs/mkdocs/docs/features/parsing/parse_exceptions.md b/docs/mkdocs/docs/features/parsing/parse_exceptions.md index 25b4768ff..6fdea5e4d 100644 --- a/docs/mkdocs/docs/features/parsing/parse_exceptions.md +++ b/docs/mkdocs/docs/features/parsing/parse_exceptions.md @@ -64,7 +64,8 @@ bool parse_error(std::size_t position, const json::exception& ex); ``` -The return value indicates whether the parsing should continue, so the function should usually return `#!cpp false`. +The return value decides whether to stop parsing (`#!cpp false`) or to repair the error and continue +(`#!cpp true`); see [error recovery](error_recovery.md) for the latter. ??? example diff --git a/docs/mkdocs/docs/features/parsing/sax_interface.md b/docs/mkdocs/docs/features/parsing/sax_interface.md index cfea680a4..3384267a0 100644 --- a/docs/mkdocs/docs/features/parsing/sax_interface.md +++ b/docs/mkdocs/docs/features/parsing/sax_interface.md @@ -60,7 +60,8 @@ bool key(string_t& val); bool parse_error(std::size_t position, const std::string& last_token, const json::exception& ex); ``` -The return value of each function determines whether parsing should proceed. +The return value of each function determines whether parsing should proceed. For `parse_error`, returning +`#!cpp true` [recovers from the error](error_recovery.md). To implement your own SAX handler, proceed as follows: @@ -68,7 +69,7 @@ To implement your own SAX handler, proceed as follows: 2. Create an object of your SAX interface class, e.g. `my_sax`. 3. Call `#!cpp bool json::sax_parse(input, &my_sax);` where the first parameter can be any input like a string or an input stream and the second parameter is a pointer to your SAX interface. -Note the `sax_parse` function only returns a `#!cpp bool` indicating the result of the last executed SAX event. It does not return `json` value - it is up to you to decide what to do with the SAX events. Furthermore, no exceptions are thrown in case of a parse error - it is up to you what to do with the exception object passed to your `parse_error` implementation. Internally, the SAX interface is used for the DOM parser (class `json_sax_dom_parser`) as well as the acceptor (`json_sax_acceptor`), see file `json_sax.hpp`. +Note the `sax_parse` function only returns a `#!cpp bool` indicating whether the input was parsed without errors and no SAX event returned `#!cpp false`. It does not return `json` value - it is up to you to decide what to do with the SAX events. Furthermore, no exceptions are thrown in case of a parse error - it is up to you what to do with the exception object passed to your `parse_error` implementation. Internally, the SAX interface is used for the DOM parser (class `json_sax_dom_parser`) as well as the acceptor (`json_sax_acceptor`), see file `json_sax.hpp`. ## See also diff --git a/docs/mkdocs/mkdocs.yml b/docs/mkdocs/mkdocs.yml index 70537bd27..22035f3cf 100644 --- a/docs/mkdocs/mkdocs.yml +++ b/docs/mkdocs/mkdocs.yml @@ -87,6 +87,7 @@ nav: - features/object_order.md - Parsing: - features/parsing/index.md + - features/parsing/error_recovery.md - features/parsing/json_lines.md - features/parsing/parse_exceptions.md - features/parsing/parser_callbacks.md diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index b0675c626..778ad5722 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -188,8 +188,8 @@ class binary_reader if (JSON_HEDLEY_UNLIKELY(current != char_traits::eof())) { - return sax->parse_error(chars_read, get_token_string(), parse_error::create(110, chars_read, - exception_message(input_format, concat("expected end of input; last byte: 0x", get_token_string()), "value"), nullptr)); + return report_error(chars_read, get_token_string(), parse_error::create(110, chars_read, + exception_message(input_format, concat("expected end of input; last byte: 0x", get_token_string()), "value"), nullptr)); } } @@ -295,8 +295,8 @@ class binary_reader { if (JSON_HEDLEY_UNLIKELY(document_size < 0 || static_cast(document_size) != chars_read - document_start)) { - return sax->parse_error(chars_read, get_token_string(), parse_error::create(112, chars_read, - exception_message(input_format_t::bson, concat("document size ", std::to_string(document_size), " does not match the number of bytes read (", std::to_string(chars_read - document_start), ")"), "document"), nullptr)); + return report_error(chars_read, get_token_string(), parse_error::create(112, chars_read, + exception_message(input_format_t::bson, concat("document size ", std::to_string(document_size), " does not match the number of bytes read (", std::to_string(chars_read - document_start), ")"), "document"), nullptr)); } return true; } @@ -490,8 +490,8 @@ class binary_reader if (JSON_HEDLEY_UNLIKELY(len < 1)) { auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, - exception_message(input_format_t::bson, concat("string length must be at least 1, is ", std::to_string(len)), "string"), nullptr)); + return report_error(chars_read, last_token, parse_error::create(112, chars_read, + exception_message(input_format_t::bson, concat("string length must be at least 1, is ", std::to_string(len)), "string"), nullptr)); } if (JSON_HEDLEY_UNLIKELY(!get_string(input_format_t::bson, len - static_cast(1), result))) @@ -502,10 +502,10 @@ class binary_reader if (JSON_HEDLEY_UNLIKELY(get() != 0x00)) { auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, - exception_message(input_format_t::bson, - "BSON string is not null-terminated", - "string"), nullptr)); + return report_error(chars_read, last_token, parse_error::create(112, chars_read, + exception_message(input_format_t::bson, + "BSON string is not null-terminated", + "string"), nullptr)); } return true; @@ -526,8 +526,8 @@ class binary_reader if (JSON_HEDLEY_UNLIKELY(len < 0)) { auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, - exception_message(input_format_t::bson, concat("byte array length cannot be negative, is ", std::to_string(len)), "binary"), nullptr)); + return report_error(chars_read, last_token, parse_error::create(112, chars_read, + exception_message(input_format_t::bson, concat("byte array length cannot be negative, is ", std::to_string(len)), "binary"), nullptr)); } // All BSON binary values have a subtype @@ -620,8 +620,8 @@ class binary_reader std::array cr{{}}; static_cast((std::snprintf)(cr.data(), cr.size(), "%.2hhX", static_cast(element_type))); // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg) const std::string cr_str{cr.data()}; - return sax->parse_error(element_type_parse_position, cr_str, - parse_error::create(114, element_type_parse_position, concat("Unsupported BSON record type 0x", cr_str), nullptr)); + return report_error(element_type_parse_position, cr_str, + parse_error::create(114, element_type_parse_position, concat("Unsupported BSON record type 0x", cr_str), nullptr)); } } } @@ -641,9 +641,9 @@ class binary_reader const auto max_val = static_cast((std::numeric_limits::max)()); if (number > max_val) { - return sax->parse_error(chars_read, get_token_string(), - parse_error::create(112, chars_read, - exception_message(input_format_t::cbor, "negative integer overflow", "value"), nullptr)); + return report_error(chars_read, get_token_string(), + parse_error::create(112, chars_read, + exception_message(input_format_t::cbor, "negative integer overflow", "value"), nullptr)); } return sax->number_integer(static_cast(-1) - static_cast(number)); } @@ -978,8 +978,8 @@ class binary_reader case cbor_tag_handler_t::error: { auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, - exception_message(input_format_t::cbor, concat("invalid byte: 0x", last_token), "value"), nullptr)); + return report_error(chars_read, last_token, parse_error::create(112, chars_read, + exception_message(input_format_t::cbor, concat("invalid byte: 0x", last_token), "value"), nullptr)); } case cbor_tag_handler_t::ignore: @@ -1177,8 +1177,8 @@ class binary_reader default: // anything else (0xFF is handled inside the other types) { auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, - exception_message(input_format_t::cbor, concat("invalid byte: 0x", last_token), "value"), nullptr)); + return report_error(chars_read, last_token, parse_error::create(112, chars_read, + exception_message(input_format_t::cbor, concat("invalid byte: 0x", last_token), "value"), nullptr)); } } } @@ -1257,8 +1257,8 @@ class binary_reader default: { auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, - exception_message(input_format_t::cbor, concat("expected length specification (0x60-0x7B) or indefinite string type (0x7F); last byte: 0x", last_token), "string"), nullptr)); + return report_error(chars_read, last_token, parse_error::create(113, chars_read, + exception_message(input_format_t::cbor, concat("expected length specification (0x60-0x7B) or indefinite string type (0x7F); last byte: 0x", last_token), "string"), nullptr)); } } } @@ -1402,8 +1402,8 @@ class binary_reader default: { auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, - exception_message(input_format_t::cbor, concat("expected length specification (0x40-0x5B) or indefinite binary array type (0x5F); last byte: 0x", last_token), "binary"), nullptr)); + return report_error(chars_read, last_token, parse_error::create(113, chars_read, + exception_message(input_format_t::cbor, concat("expected length specification (0x40-0x5B) or indefinite binary array type (0x5F); last byte: 0x", last_token), "binary"), nullptr)); } } } @@ -1483,8 +1483,8 @@ class binary_reader { if (JSON_HEDLEY_UNLIKELY(!value_in_range_of(len) || len == detail::unknown_size())) { - return sax->parse_error(chars_read, get_token_string(), out_of_range::create(408, - exception_message(input_format_t::cbor, concat("excessive ", context, " size"), "size"), nullptr)); + return report_error(chars_read, get_token_string(), out_of_range::create(408, + exception_message(input_format_t::cbor, concat("excessive ", context, " size"), "size"), nullptr)); } result = conditional_static_cast(len); return true; @@ -1980,8 +1980,8 @@ class binary_reader default: // anything else { auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, - exception_message(input_format_t::msgpack, concat("invalid byte: 0x", last_token), "value"), nullptr)); + return report_error(chars_read, last_token, parse_error::create(112, chars_read, + exception_message(input_format_t::msgpack, concat("invalid byte: 0x", last_token), "value"), nullptr)); } } } @@ -2063,8 +2063,8 @@ class binary_reader default: { auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, - exception_message(input_format_t::msgpack, concat("expected length specification (0xA0-0xBF, 0xD9-0xDB); last byte: 0x", last_token), "string"), nullptr)); + return report_error(chars_read, last_token, parse_error::create(113, chars_read, + exception_message(input_format_t::msgpack, concat("expected length specification (0xA0-0xBF, 0xD9-0xDB); last byte: 0x", last_token), "string"), nullptr)); } } } @@ -2381,8 +2381,8 @@ class binary_reader { if (JSON_HEDLEY_UNLIKELY(len < 0)) { - return sax->parse_error(chars_read, get_token_string(), parse_error::create(113, chars_read, - exception_message(input_format, "string length must not be negative", "string"), nullptr)); + return report_error(chars_read, get_token_string(), parse_error::create(113, chars_read, + exception_message(input_format, "string length must not be negative", "string"), nullptr)); } return true; } @@ -2493,7 +2493,7 @@ class binary_reader { message = "expected length type specification (U, i, u, I, m, l, M, L); last byte: 0x" + last_token; } - return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format, message, "string"), nullptr)); + return report_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format, message, "string"), nullptr)); } /*! @@ -2594,8 +2594,8 @@ class binary_reader } if (number < 0) { - return sax->parse_error(chars_read, get_token_string(), parse_error::create(113, chars_read, - exception_message(input_format, "count in an optimized container must be positive", "size"), nullptr)); + return report_error(chars_read, get_token_string(), parse_error::create(113, chars_read, + exception_message(input_format, "count in an optimized container must be positive", "size"), nullptr)); } result = static_cast(number); // NOLINT(bugprone-signed-char-misuse,cert-str34-c): number is not a char return true; @@ -2610,8 +2610,8 @@ class binary_reader } if (number < 0) { - return sax->parse_error(chars_read, get_token_string(), parse_error::create(113, chars_read, - exception_message(input_format, "count in an optimized container must be positive", "size"), nullptr)); + return report_error(chars_read, get_token_string(), parse_error::create(113, chars_read, + exception_message(input_format, "count in an optimized container must be positive", "size"), nullptr)); } result = static_cast(number); return true; @@ -2626,8 +2626,8 @@ class binary_reader } if (number < 0) { - return sax->parse_error(chars_read, get_token_string(), parse_error::create(113, chars_read, - exception_message(input_format, "count in an optimized container must be positive", "size"), nullptr)); + return report_error(chars_read, get_token_string(), parse_error::create(113, chars_read, + exception_message(input_format, "count in an optimized container must be positive", "size"), nullptr)); } result = static_cast(number); return true; @@ -2642,13 +2642,13 @@ class binary_reader } if (number < 0) { - return sax->parse_error(chars_read, get_token_string(), parse_error::create(113, chars_read, - exception_message(input_format, "count in an optimized container must be positive", "size"), nullptr)); + return report_error(chars_read, get_token_string(), parse_error::create(113, chars_read, + exception_message(input_format, "count in an optimized container must be positive", "size"), nullptr)); } if (!value_in_range_of(number)) { - return sax->parse_error(chars_read, get_token_string(), out_of_range::create(408, - exception_message(input_format, "integer value overflow", "size"), nullptr)); + return report_error(chars_read, get_token_string(), out_of_range::create(408, + exception_message(input_format, "integer value overflow", "size"), nullptr)); } result = static_cast(number); return true; @@ -2697,8 +2697,8 @@ class binary_reader } if (!value_in_range_of(number)) { - return sax->parse_error(chars_read, get_token_string(), out_of_range::create(408, - exception_message(input_format, "integer value overflow", "size"), nullptr)); + return report_error(chars_read, get_token_string(), out_of_range::create(408, + exception_message(input_format, "integer value overflow", "size"), nullptr)); } result = detail::conditional_static_cast(number); return true; @@ -2712,7 +2712,7 @@ class binary_reader } if (is_ndarray) // ndarray dimensional vector can only contain integers and cannot embed another array { - return sax->parse_error(chars_read, get_token_string(), parse_error::create(113, chars_read, exception_message(input_format, "ndarray dimensional vector is not allowed", "size"), nullptr)); + return report_error(chars_read, get_token_string(), parse_error::create(113, chars_read, exception_message(input_format, "ndarray dimensional vector is not allowed", "size"), nullptr)); } std::vector dim; if (JSON_HEDLEY_UNLIKELY(!get_ubjson_ndarray_size(dim))) @@ -2748,13 +2748,13 @@ class binary_reader // as modular arithmetic can produce any value, not just 0 or SIZE_MAX. if (JSON_HEDLEY_UNLIKELY(i > 0 && result > (std::numeric_limits::max)() / i)) { - return sax->parse_error(chars_read, get_token_string(), out_of_range::create(408, exception_message(input_format, "excessive ndarray size caused overflow", "size"), nullptr)); + return report_error(chars_read, get_token_string(), out_of_range::create(408, exception_message(input_format, "excessive ndarray size caused overflow", "size"), nullptr)); } result *= i; // Additional post-multiplication check to catch any edge cases the pre-check might miss if (result == 0 || result == npos) { - return sax->parse_error(chars_read, get_token_string(), out_of_range::create(408, exception_message(input_format, "excessive ndarray size caused overflow", "size"), nullptr)); + return report_error(chars_read, get_token_string(), out_of_range::create(408, exception_message(input_format, "excessive ndarray size caused overflow", "size"), nullptr)); } if (JSON_HEDLEY_UNLIKELY(!sax->number_unsigned(static_cast(i)))) { @@ -2782,7 +2782,7 @@ class binary_reader { message = "expected length type specification (U, i, u, I, m, l, M, L) after '#'; last byte: 0x" + last_token; } - return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format, message, "size"), nullptr)); + return report_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format, message, "size"), nullptr)); } /*! @@ -2816,8 +2816,8 @@ class binary_reader && JSON_HEDLEY_UNLIKELY(std::binary_search(bjd_optimized_type_markers.begin(), bjd_optimized_type_markers.end(), result.second))) { auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, - exception_message(input_format, concat("marker 0x", last_token, " is not a permitted optimized array type"), "type"), nullptr)); + return report_error(chars_read, last_token, parse_error::create(112, chars_read, + exception_message(input_format, concat("marker 0x", last_token, " is not a permitted optimized array type"), "type"), nullptr)); } if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "type"))) @@ -2833,8 +2833,8 @@ class binary_reader return false; } auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, - exception_message(input_format, concat("expected '#' after type information; last byte: 0x", last_token), "size"), nullptr)); + return report_error(chars_read, last_token, parse_error::create(112, chars_read, + exception_message(input_format, concat("expected '#' after type information; last byte: 0x", last_token), "size"), nullptr)); } const bool is_error = get_ubjson_size_value(result.first, is_ndarray); @@ -2853,8 +2853,8 @@ class binary_reader const bool is_error = get_ubjson_size_value(result.first, is_ndarray); if (input_format == input_format_t::bjdata && is_ndarray && !inside_ndarray) { - return sax->parse_error(chars_read, get_token_string(), parse_error::create(112, chars_read, - exception_message(input_format, "ndarray requires both type and size", "size"), nullptr)); + return report_error(chars_read, get_token_string(), parse_error::create(112, chars_read, + exception_message(input_format, "ndarray requires both type and size", "size"), nullptr)); } return is_error; } @@ -3030,8 +3030,8 @@ class binary_reader if (JSON_HEDLEY_UNLIKELY(current > 127)) { auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, - exception_message(input_format, concat("byte after 'C' must be in range 0x00..0x7F; last byte: 0x", last_token), "char"), nullptr)); + return report_error(chars_read, last_token, parse_error::create(113, chars_read, + exception_message(input_format, concat("byte after 'C' must be in range 0x00..0x7F; last byte: 0x", last_token), "char"), nullptr)); } string_t s(1, static_cast(current)); return sax->string(s); @@ -3053,7 +3053,7 @@ class binary_reader break; } auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format, "invalid byte: 0x" + last_token, "value"), nullptr)); + return report_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format, "invalid byte: 0x" + last_token, "value"), nullptr)); } /*! @@ -3081,8 +3081,8 @@ class binary_reader if (JSON_HEDLEY_UNLIKELY(it == bjd_types_map.end() || it->first != size_and_type.second)) { auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, - exception_message(input_format, "invalid byte: 0x" + last_token, "type"), nullptr)); + return report_error(chars_read, last_token, parse_error::create(112, chars_read, + exception_message(input_format, "invalid byte: 0x" + last_token, "type"), nullptr)); } string_t type = it->second; // sax->string() takes a reference @@ -3129,8 +3129,8 @@ class binary_reader if (JSON_HEDLEY_UNLIKELY((size_and_type.second == 'Z' || size_and_type.second == 'T' || size_and_type.second == 'F') && size_and_type.first > max_valueless_container_size)) { - return sax->parse_error(chars_read, get_token_string(), out_of_range::create(408, - exception_message(input_format, "excessive array size", "size"), nullptr)); + return report_error(chars_read, get_token_string(), out_of_range::create(408, + exception_message(input_format, "excessive array size", "size"), nullptr)); } if (JSON_HEDLEY_UNLIKELY(!enter_array(size_and_type.first, size_and_type.second))) @@ -3166,8 +3166,8 @@ class binary_reader if (input_format == input_format_t::bjdata && size_and_type.first != npos && (size_and_type.second & (1 << 8)) != 0) { auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, - exception_message(input_format, "BJData object does not support ND-array size in optimized format", "object"), nullptr)); + return report_error(chars_read, last_token, parse_error::create(112, chars_read, + exception_message(input_format, "BJData object does not support ND-array size in optimized format", "object"), nullptr)); } if (size_and_type.first != npos) @@ -3215,8 +3215,8 @@ class binary_reader if (JSON_HEDLEY_UNLIKELY(result_remainder != token_type::end_of_input)) { - return sax->parse_error(chars_read, number_string, parse_error::create(115, chars_read, - exception_message(input_format, concat("invalid number text: ", number_lexer.get_token_string()), "high-precision number"), nullptr)); + return report_error(chars_read, number_string, parse_error::create(115, chars_read, + exception_message(input_format, concat("invalid number text: ", number_lexer.get_token_string()), "high-precision number"), nullptr)); } switch (result_number) @@ -3230,7 +3230,7 @@ class binary_reader const auto parsed_float = number_lexer.get_number_float(); if (JSON_HEDLEY_UNLIKELY(!std::isfinite(parsed_float))) { - return sax->parse_error( + return report_error( chars_read, number_string, out_of_range::create(406, concat("number overflow parsing '", number_string, '\''), nullptr)); @@ -3255,8 +3255,8 @@ class binary_reader case token_type::end_of_input: case token_type::literal_or_value: default: - return sax->parse_error(chars_read, number_string, parse_error::create(115, chars_read, - exception_message(input_format, concat("invalid number text: ", number_lexer.get_token_string()), "high-precision number"), nullptr)); + return report_error(chars_read, number_string, parse_error::create(115, chars_read, + exception_message(input_format, concat("invalid number text: ", number_lexer.get_token_string()), "high-precision number"), nullptr)); } } @@ -3324,8 +3324,8 @@ class binary_reader bool bon8_error(const std::string& detail, const char* context) { auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, - exception_message(input_format_t::bon8, concat(detail, ": 0x", last_token), context), nullptr)); + return report_error(chars_read, last_token, parse_error::create(112, chars_read, + exception_message(input_format_t::bon8, concat(detail, ": 0x", last_token), context), nullptr)); } /*! @@ -3839,8 +3839,7 @@ class binary_reader { // in case of failure, advance position by 1 to report the failing location ++chars_read; - sax->parse_error(chars_read, "", parse_error::create(110, chars_read, exception_message(format, "unexpected end of input", context), nullptr)); - return false; + return report_error(chars_read, "", parse_error::create(110, chars_read, exception_message(format, "unexpected end of input", context), nullptr)); } return true; } @@ -3952,9 +3951,9 @@ class binary_reader // (which would defeat allow_exceptions=false / strict discarding). if (JSON_HEDLEY_UNLIKELY(!is_valid_utf8(result, old_size))) { - return sax->parse_error(chars_read, get_token_string(), - parse_error::create(113, chars_read, - exception_message(format, "invalid string: ill-formed UTF-8 byte", "string"), nullptr)); + return report_error(chars_read, get_token_string(), + parse_error::create(113, chars_read, + exception_message(format, "invalid string: ill-formed UTF-8 byte", "string"), nullptr)); } return true; @@ -4041,6 +4040,25 @@ class binary_reader return true; } + /*! + @brief report an error to the SAX parser + + The binary formats cannot recover from an error: a value's size is given + before its payload, and every byte value is a valid type marker, so after + an error there is no way to find where the next value begins. Reading + therefore stops, whatever the SAX parser's parse_error() returns. That the + SAX parser may ask for the containers read so far to be closed is handled + by @ref json_sax_salvager, not here (see #3989). + + @return false, so that the caller stops reading + */ + template + bool report_error(const std::size_t position, const std::string& last_token, const Exception& ex) const + { + static_cast(sax->parse_error(position, last_token, ex)); + return false; + } + /*! @param[in] format the current format (for diagnostics) @param[in] context further context information (for diagnostics) @@ -4051,8 +4069,8 @@ class binary_reader { if (JSON_HEDLEY_UNLIKELY(current == char_traits::eof())) { - return sax->parse_error(chars_read, "", - parse_error::create(110, chars_read, exception_message(format, "unexpected end of input", context), nullptr)); + return report_error(chars_read, "", + parse_error::create(110, chars_read, exception_message(format, "unexpected end of input", context), nullptr)); } return true; } diff --git a/include/nlohmann/detail/input/json_sax.hpp b/include/nlohmann/detail/input/json_sax.hpp index 962913610..7af6e1fe4 100644 --- a/include/nlohmann/detail/input/json_sax.hpp +++ b/include/nlohmann/detail/input/json_sax.hpp @@ -131,7 +131,9 @@ struct json_sax @param[in] position the position in the input where the error occurs @param[in] last_token the last read token @param[in] ex an exception object describing the error - @return whether parsing should proceed (must return false) + @return whether to recover from the error: false stops parsing; true + repairs JSON text and continues, or, for the binary formats, stops + after closing the containers read so far */ virtual bool parse_error(std::size_t position, const std::string& last_token, @@ -186,9 +188,12 @@ a pointer to the respective array or object for each recursion depth. After successful parsing, the value that is passed by reference to the constructor contains the parsed value. -@tparam BasicJsonType the JSON type +@tparam BasicJsonType the JSON type +@tparam InputAdapterType the input adapter of the lexer that can be passed to + the constructor to record diagnostic positions; it + does not matter if no lexer is passed */ -template +template class json_sax_dom_parser { public: @@ -505,7 +510,7 @@ class json_sax_dom_parser lexer_t* m_lexer_ref = nullptr; }; -template +template class json_sax_dom_callback_parser { public: @@ -1207,5 +1212,176 @@ class json_sax_acceptor } }; +/*! +@brief SAX proxy that lets the binary readers keep what was read before an error + +The binary formats cannot continue after an error: a value's size is given +before its payload, and every byte value is a valid type marker, so there is no +way to find where the next value begins. When the SAX parser's parse_error() +returns true to ask for error recovery, the best the binary readers can offer is +the value read up to the error. + +This proxy forwards every event to the SAX parser and records which containers +are open and whether a key still waits for its value. After an error the SAX +parser asked to recover from, @ref close_open_containers then completes the +value with null for a pending key and the missing end events, so the SAX parser +sees balanced events (see #3989). + +@tparam BasicJsonType the JSON type +@tparam SAX the SAX parser to forward the events to +*/ +template +class json_sax_salvager +{ + public: + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using binary_t = typename BasicJsonType::binary_t; + + explicit json_sax_salvager(SAX* sax_) noexcept + : sax(sax_) + {} + + bool null() + { + key_pending = false; + return sax->null(); + } + + bool boolean(bool val) + { + key_pending = false; + return sax->boolean(val); + } + + bool number_integer(number_integer_t val) + { + key_pending = false; + return sax->number_integer(val); + } + + bool number_unsigned(number_unsigned_t val) + { + key_pending = false; + return sax->number_unsigned(val); + } + + bool number_float(number_float_t val, const string_t& s) + { + key_pending = false; + return sax->number_float(val, s); + } + + bool string(string_t& val) + { + key_pending = false; + return sax->string(val); + } + + bool binary(binary_t& val) + { + key_pending = false; + return sax->binary(val); + } + + bool start_object(std::size_t len) + { + key_pending = false; + if (JSON_HEDLEY_UNLIKELY(!sax->start_object(len))) + { + return false; + } + open_containers.push_back(true); + return true; + } + + bool key(string_t& val) + { + key_pending = true; + return sax->key(val); + } + + bool end_object() + { + JSON_ASSERT(!open_containers.empty() && open_containers.back()); + open_containers.pop_back(); + return sax->end_object(); + } + + bool start_array(std::size_t len) + { + key_pending = false; + if (JSON_HEDLEY_UNLIKELY(!sax->start_array(len))) + { + return false; + } + open_containers.push_back(false); + return true; + } + + bool end_array() + { + JSON_ASSERT(!open_containers.empty() && !open_containers.back()); + open_containers.pop_back(); + return sax->end_array(); + } + + template + bool parse_error(std::size_t position, const std::string& last_token, + const Exception& ex) + { + recovery_requested = sax->parse_error(position, last_token, ex); + // the binary readers stop after an error anyway + return false; + } + + /*! + @brief complete the value read before an error + + Does nothing unless the SAX parser's parse_error() returned true. Otherwise + passes null for a key that waits for its value and closes the containers + that are still open, innermost first, until an event returns false. + */ + void close_open_containers() + { + if (!recovery_requested) + { + return; + } + recovery_requested = false; + + if (key_pending) + { + key_pending = false; + if (JSON_HEDLEY_UNLIKELY(!sax->null())) + { + return; + } + } + + while (!open_containers.empty()) + { + const bool is_object = open_containers.back(); + open_containers.pop_back(); + if (JSON_HEDLEY_UNLIKELY(is_object ? !sax->end_object() : !sax->end_array())) + { + return; + } + } + } + + private: + /// the SAX parser the events are forwarded to + SAX* sax = nullptr; + /// the containers that are open, innermost last; true for an object + std::vector open_containers {}; // NOLINT(readability-redundant-member-init) + /// whether a key was passed whose value has not been passed yet + bool key_pending = false; + /// whether the SAX parser's parse_error() asked to recover from the error + bool recovery_requested = false; +}; + } // namespace detail NLOHMANN_JSON_NAMESPACE_END diff --git a/include/nlohmann/detail/input/lexer.hpp b/include/nlohmann/detail/input/lexer.hpp index 98c0fd76a..ed48359f2 100644 --- a/include/nlohmann/detail/input/lexer.hpp +++ b/include/nlohmann/detail/input/lexer.hpp @@ -11,6 +11,7 @@ #include // array #include // localeconv #include // size_t +#include // uint8_t #include // snprintf #include // strtof, strtod, strtold, strtoll, strtoull #include // initializer_list @@ -453,8 +454,16 @@ class lexer : public lexer_base if (0xD800 <= codepoint1 && codepoint1 <= 0xDBFF) { // expect next \uxxxx entry - if (JSON_HEDLEY_LIKELY(get() == '\\' && get() == 'u')) + if (JSON_HEDLEY_LIKELY(get() == '\\')) { + if (JSON_HEDLEY_UNLIKELY(get() != 'u')) + { + // current is the character escaped by the backslash + error_message = "invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF"; + string_error_resume = resume_kind::escaped_character; + return token_type::parse_error; + } + const int codepoint2 = get_codepoint(); if (JSON_HEDLEY_UNLIKELY(codepoint2 == -1)) @@ -479,7 +488,11 @@ class lexer : public lexer_base } else { + // the second escape was read completely and is a + // code point of its own error_message = "invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF"; + string_error_resume = resume_kind::after_escape; + string_error_codepoint = codepoint2; return token_type::parse_error; } } @@ -493,7 +506,9 @@ class lexer : public lexer_base { if (JSON_HEDLEY_UNLIKELY(0xDC00 <= codepoint1 && codepoint1 <= 0xDFFF)) { + // the escape was read completely error_message = "invalid string: surrogate U+DC00..U+DFFF must follow U+D800..U+DBFF"; + string_error_resume = resume_kind::after_escape; return token_type::parse_error; } } @@ -2238,6 +2253,552 @@ scan_number_done: } } + ///////////////////// + // error recovery + ///////////////////// + + /*! + @brief make the best of the token that scan() rejected + + Called by the parser after scan() returned token_type::parse_error and the + SAX parser asked to recover from the error (see #3989). Keeps what can be + read of the token and skips the rest: + + - A string keeps its characters. An unknown escape stands for the escaped + character itself (as in JavaScript), an invalid `\u` escape and ill-formed + UTF-8 become U+FFFD, and a control character is kept. A line break or the + end of the input ends a string that lacks its closing quote. + - A number keeps its longest valid prefix, e.g. `1` for `1.` or `1e+`. + - A block comment that is not closed runs to the end of the input. + - Anything else is skipped. + + The rest of an invalid token is skipped up to the next delimiter + (whitespace, a structural character, or a quote). A delimiter that the + invalid token consumed is returned to the input, so that the next scan() + reads it. + + @return token_type::value_string or a number token type if a string or a + number could be read, token_type::end_of_input for a block comment + that is not closed, token_type::uninitialized otherwise + */ + token_type recover_token() + { + const resume_kind resume = string_error_resume; + const int codepoint = string_error_codepoint; + string_error_resume = resume_kind::character; + string_error_codepoint = -1; + + if (error_message_starts_with("invalid string")) + { + return recover_string(resume, codepoint); + } + + if (error_message_starts_with("invalid number")) + { + return recover_number(); + } + + if (error_message_starts_with("invalid comment; missing")) + { + // the comment runs to the end of the input + return token_type::end_of_input; + } + + skip_to_delimiter(); + return token_type::uninitialized; + } + + /*! + @brief return the token that scan() read last to the input, so that the + next scan() reads it again + + Called by the parser when recovering from an error. The token must be a + single character (',', ':', '[', ']', '{', or '}') or the end of the + input, and scan() must have read it last. + */ + void unget_token() + { + JSON_ASSERT(!next_unget); + unget(); + } + + /*! + @brief let the token string for the next error begin at the current character + + The token string of an error reaches back to the beginning of the last + string or number. After an error, the parser calls this function so that + the next error does not report (and, with many errors, copy) everything + read since then. + */ + void restart_token_string() + { + restart_token_string_impl(std::integral_constant {}); + } + + private: + /// how recover_string() continues after the error scan_string() reported + enum class resume_kind : std::uint8_t + { + /// current is the next character of the string (or the end of input) + character, + /// current is the character escaped by the preceding backslash + escaped_character, + /// current is the last character of a complete escape + after_escape + }; + + /// whether error_message begins with @a prefix + bool error_message_starts_with(const char* prefix) const noexcept + { + const char* message = error_message; + while (*prefix != '\0') + { + if (*message++ != *prefix++) + { + return false; + } + } + return true; + } + + /// whether current ends an invalid token (see recover_token()) + bool current_is_delimiter() const noexcept + { + switch (current) + { + case ' ': + case '\t': + case '\n': + case '\r': + case '[': + case ']': + case '{': + case '}': + case ',': + case ':': + case '\"': +#if !JSON_STRICT_NUL_HANDLING + case '\0': +#endif + case char_traits::eof(): + return true; + + case '/': + return ignore_comments; + + default: + return false; + } + } + + /// skip the rest of an invalid token and return its delimiter to the input + void skip_to_delimiter() + { + while (!current_is_delimiter()) + { + get(); + } + + if (current != char_traits::eof()) + { + unget(); + } + } + + /// append U+FFFD REPLACEMENT CHARACTER to token_buffer + void add_replacement_character() + { + add(0xEF); + add(0xBF); + add(0xBD); + } + + /// append the UTF-8 encoding of @a codepoint (not a surrogate) to token_buffer + void add_codepoint(const int codepoint) + { + JSON_ASSERT(0x00 <= codepoint && codepoint <= 0x10FFFF); + const auto cp = static_cast(codepoint); + if (cp < 0x80) + { + add(static_cast(cp)); + } + else if (cp <= 0x7FF) + { + add(static_cast(0xC0u | (cp >> 6u))); + add(static_cast(0x80u | (cp & 0x3Fu))); + } + else if (cp <= 0xFFFF) + { + add(static_cast(0xE0u | (cp >> 12u))); + add(static_cast(0x80u | ((cp >> 6u) & 0x3Fu))); + add(static_cast(0x80u | (cp & 0x3Fu))); + } + else + { + add(static_cast(0xF0u | (cp >> 18u))); + add(static_cast(0x80u | ((cp >> 12u) & 0x3Fu))); + add(static_cast(0x80u | ((cp >> 6u) & 0x3Fu))); + add(static_cast(0x80u | (cp & 0x3Fu))); + } + } + + /// append a code point read from a `\u` escape; a surrogate becomes U+FFFD + void add_escaped_codepoint(const int codepoint) + { + if (0xD800 <= codepoint && codepoint <= 0xDFFF) + { + add_replacement_character(); + } + else + { + add_codepoint(codepoint); + } + } + + /*! + @brief remove an incomplete UTF-8 sequence from the end of token_buffer + + next_byte_in_range() adds the bytes of a sequence as it checks them, so + when it rejects a byte, the beginning of the sequence is already in + token_buffer, which otherwise holds only complete sequences. + + @return whether an incomplete sequence was removed + */ + bool remove_incomplete_utf8_sequence() + { + std::size_t lead = token_buffer.size(); + std::size_t continuation_bytes = 0; + while (lead > 0 && continuation_bytes < 3 + && (static_cast(token_buffer[lead - 1]) & 0xC0u) == 0x80u) + { + --lead; + ++continuation_bytes; + } + if (lead == 0) + { + return false; + } + + const auto lead_byte = static_cast(token_buffer[lead - 1]); + const std::size_t expected = (lead_byte >= 0xF0) ? 3 : (lead_byte >= 0xE0) ? 2 : (lead_byte >= 0xC0) ? 1 : 0; + if (continuation_bytes >= expected) + { + return false; + } + + token_buffer.resize(lead - 1); + return true; + } + + /*! + @brief read the UTF-8 sequence that begins with current, which is not ASCII + @return whether the next character must be read; false if current still + needs to be handled, because it does not belong to the sequence + */ + bool recover_utf8_sequence() + { + // the number of continuation bytes and the range of the first one; + // see the ranges in scan_string() + std::size_t count = 0; + char_int_type low = 0x80; + char_int_type high = 0xBF; + if (current >= 0xC2 && current <= 0xDF) + { + count = 1; + } + else if (current >= 0xE0 && current <= 0xEF) + { + count = 2; + low = (current == 0xE0) ? 0xA0 : 0x80; + high = (current == 0xED) ? 0x9F : 0xBF; + } + else if (current >= 0xF0 && current <= 0xF4) + { + count = 3; + low = (current == 0xF0) ? 0x90 : 0x80; + high = (current == 0xF4) ? 0x8F : 0xBF; + } + else + { + // an ill-formed byte + add_replacement_character(); + return true; + } + + const std::size_t start = token_buffer.size(); + add(current); + for (std::size_t i = 0; i < count; ++i) + { + get(); + if (current < low || current > high) + { + token_buffer.resize(start); + add_replacement_character(); + return false; + } + add(current); + low = 0x80; + high = 0xBF; + } + return true; + } + + /*! + @brief read the low surrogate that must follow the high surrogate @a high + @return whether the next character must be read; false if current still + needs to be handled + */ + bool recover_low_surrogate(int high) + { + while (true) + { + if (get() != '\\') + { + add_replacement_character(); + return false; + } + if (get() != 'u') + { + add_replacement_character(); + // not 'u', so this does not come back here + return recover_escape(); + } + + const int low = get_codepoint(); + if (low == -1) + { + add_replacement_character(); + return false; + } + if (0xDC00 <= low && low <= 0xDFFF) + { + add_codepoint(static_cast((static_cast(high) << 10u) + + static_cast(low) - 0x35FDC00u)); + return true; + } + + // high has no low surrogate + add_replacement_character(); + if (low < 0xD800 || low > 0xDBFF) + { + add_codepoint(low); + return true; + } + // another high surrogate + high = low; + } + } + + /*! + @brief read the escape whose backslash was read; current is the escaped character + @return whether the next character must be read; false if current still + needs to be handled + */ + bool recover_escape() + { + switch (current) + { + case '\"': + add('\"'); + return true; + case '\\': + add('\\'); + return true; + case '/': + add('/'); + return true; + case 'b': + add('\b'); + return true; + case 'f': + add('\f'); + return true; + case 'n': + add('\n'); + return true; + case 'r': + add('\r'); + return true; + case 't': + add('\t'); + return true; + + case 'u': + { + const int codepoint = get_codepoint(); + if (codepoint == -1) + { + add_replacement_character(); + return false; + } + if (0xD800 <= codepoint && codepoint <= 0xDBFF) + { + return recover_low_surrogate(codepoint); + } + add_escaped_codepoint(codepoint); + return true; + } + + // an unknown escape stands for the escaped character + default: + return false; + } + } + + /*! + @brief read the rest of a string after scan_string() rejected it + + token_buffer holds what scan_string() read before the error. See + recover_token() for how errors are repaired. + + @param[in] resume how to continue, see resume_kind + @param[in] codepoint for a high surrogate followed by an escape of another + code point: that code point; -1 otherwise + */ + token_type recover_string(const resume_kind resume, const int codepoint) + { + // whether the next character must be read before it can be handled + bool fetch = false; + + if (error_message_starts_with("invalid string: surrogate") + || error_message_starts_with("invalid string: '\\u'")) + { + add_replacement_character(); + } + else if (error_message_starts_with("invalid string: ill-formed UTF-8") + && remove_incomplete_utf8_sequence()) + { + add_replacement_character(); + } + + switch (resume) + { + case resume_kind::escaped_character: + fetch = recover_escape(); + break; + case resume_kind::after_escape: + if (0xD800 <= codepoint && codepoint <= 0xDBFF) + { + fetch = recover_low_surrogate(codepoint); + } + else + { + if (codepoint != -1) + { + add_escaped_codepoint(codepoint); + } + fetch = true; + } + break; + case resume_kind::character: + default: + break; + } + + while (true) + { + if (fetch) + { + get(); + } + fetch = true; + + switch (current) + { + case '\"': + // a line break or the end of the input ends a string that + // lacks its closing quote + case '\n': + case '\r': + case char_traits::eof(): + return token_type::value_string; + +#if !JSON_STRICT_NUL_HANDLING + case '\0': + // the end of the input, see scan() + unget(); + return token_type::value_string; +#endif + + case '\\': + get(); + fetch = recover_escape(); + break; + + default: + if (current < 0x80) + { + // including control characters + add(current); + } + else + { + fetch = recover_utf8_sequence(); + } + break; + } + } + } + + /*! + @brief keep the longest valid prefix of a number that scan_number() rejected + + token_buffer holds the characters scan_number() accepted before the error, + so the prefix ends at its last digit. + */ + token_type recover_number() + { + while (!token_buffer.empty() && (token_buffer.back() < '0' || token_buffer.back() > '9')) + { + token_buffer.pop_back(); + } + + if (token_buffer.empty()) + { + skip_to_delimiter(); + return token_type::uninitialized; + } + + if (decimal_point_position >= token_buffer.size()) + { + decimal_point_position = std::string::npos; + } + + const std::size_t exponent = token_buffer.find_first_of("eE"); + const std::size_t mantissa_end = (exponent == std::string::npos) ? token_buffer.size() : exponent; + token_type number_type = token_type::value_unsigned; + if (decimal_point_position != std::string::npos || exponent != std::string::npos) + { + number_type = token_type::value_float; + } + else if (token_buffer.front() == '-') + { + number_type = token_type::value_integer; + } + + const token_type result = convert_number(number_type, mantissa_end); + skip_to_delimiter(); + return result; + } + + /// seekable adapter: the token string begins at current, which was consumed + void restart_token_string_impl(std::true_type /*lazy*/) noexcept + { + const std::size_t consumed = ia.get_consumed_count(); + token_string_start = (consumed > 0 && current != char_traits::eof()) ? consumed - 1 : consumed; + } + + /// streaming adapter: the token string begins at current; a character + /// that was put back is copied again when it is read again + void restart_token_string_impl(std::false_type /*lazy*/) + { + token_string.clear(); + if (!next_unget && current != char_traits::eof()) + { + token_string.push_back(char_traits::to_char_type(current)); + } + } + private: /// input adapter InputAdapterType ia; @@ -2278,6 +2839,13 @@ scan_number_done: /// a description of occurred lexer errors const char* error_message = ""; + /// how recover_token() continues a string that scan_string() rejected; + /// set only on the error paths that need more than error_message + resume_kind string_error_resume = resume_kind::character; + /// the code point of the second escape when a high surrogate is followed + /// by an escape that is not a low surrogate; -1 otherwise + int string_error_codepoint = -1; + // number values number_integer_t value_integer = 0; number_unsigned_t value_unsigned = 0; diff --git a/include/nlohmann/detail/input/parser.hpp b/include/nlohmann/detail/input/parser.hpp index 5fec57a70..93d5ef834 100644 --- a/include/nlohmann/detail/input/parser.hpp +++ b/include/nlohmann/detail/input/parser.hpp @@ -98,7 +98,7 @@ class parser if (callback) { json_sax_dom_callback_parser sdp(result, callback, allow_exceptions, &m_lexer); - sax_parse_internal(&sdp); + sax_parse_internal(&sdp); if (strict) { @@ -135,7 +135,7 @@ class parser else { json_sax_dom_parser sdp(result, allow_exceptions, &m_lexer); - sax_parse_internal(&sdp); + sax_parse_internal(&sdp); if (strict) { @@ -173,26 +173,59 @@ class parser bool accept(const bool strict = true) { json_sax_acceptor sax_acceptor; - return sax_parse(&sax_acceptor, strict); + return sax_parse_impl(&sax_acceptor, strict); } + /*! + @brief public SAX interface + + If the SAX parser's parse_error() returns true, the parser recovers from + the error: it repairs the input and continues (see #3989). + + @param[in] sax the SAX parser + @param[in] strict whether to expect the last token to be EOF + @return whether the input was parsed without errors and no SAX event + returned false + */ template JSON_HEDLEY_NON_NULL(2) bool sax_parse(SAX* sax, const bool strict = true) + { + return sax_parse_impl(sax, strict); + } + + private: + /// what sax_parse_internal() does after an object key was expected + enum class next_step : std::uint8_t + { + /// stop parsing + stop, + /// parse a value that begins with last_token + parse_value, + /// evaluate the state of the innermost container, which reads + /// last_token again + evaluate_state + }; + + template + JSON_HEDLEY_NON_NULL(2) + bool sax_parse_impl(SAX* sax, const bool strict) { (void)detail::is_sax_static_asserts {}; - const bool result = sax_parse_internal(sax); + const bool result = sax_parse_internal(sax); if (result) { if (strict) { - // strict mode: next byte must be EOF - if (get_token() != token_type::end_of_input) + // strict mode: next byte must be EOF; after recovering from an + // error, the end of the input may already have been read + if (last_token != token_type::end_of_input && get_token() != token_type::end_of_input) { - return sax->parse_error(m_lexer.get_position(), - m_lexer.get_token_string(), - parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_of_input, "value"), nullptr)); + // the value is complete, so there is nothing to recover + static_cast(report_error(sax, parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_of_input, "value"), nullptr), + std::integral_constant {})); + return false; } } else @@ -203,14 +236,23 @@ class parser } } - return result; + return result && !error_reported; } - private: - template + /*! + @brief parse a JSON value and pass it to a SAX parser + + @tparam AllowRecovery whether to recover from an error if the SAX parser's + parse_error() returns true; false for the SAX parsers + of parse() and accept(), which never do, so that no + code for recovering is generated for them + */ + template JSON_HEDLEY_NON_NULL(2) bool sax_parse_internal(SAX* sax) { + const std::integral_constant allow_recovery{}; + // stack to remember the hierarchy of structured values we are parsing // true = array; false = object std::vector states; @@ -241,12 +283,18 @@ class parser break; } - // parse key + // remember we are now inside an object + states.push_back(false); + + // parse key (the steps of parse_key(), which are + // repeated here and below for speed) if (JSON_HEDLEY_UNLIKELY(last_token != token_type::value_string)) { - return sax->parse_error(m_lexer.get_position(), - m_lexer.get_token_string(), - parse_error::create(101, m_lexer.get_position(), exception_message(token_type::value_string, "object key"), nullptr)); + if (!continue_after(key_error(sax, allow_recovery, false), skip_to_state_evaluation)) + { + return false; + } + continue; } if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string()))) { @@ -256,14 +304,13 @@ class parser // parse separator (:) if (JSON_HEDLEY_UNLIKELY(get_token() != token_type::name_separator)) { - return sax->parse_error(m_lexer.get_position(), - m_lexer.get_token_string(), - parse_error::create(101, m_lexer.get_position(), exception_message(token_type::name_separator, "object separator"), nullptr)); + if (!continue_after(key_error(sax, allow_recovery, true), skip_to_state_evaluation)) + { + return false; + } + continue; } - // remember we are now inside an object - states.push_back(false); - // parse values get_token(); continue; @@ -299,9 +346,11 @@ class parser if (JSON_HEDLEY_UNLIKELY(!std::isfinite(res))) { - return sax->parse_error(m_lexer.get_position(), - m_lexer.get_token_string(), - out_of_range::create(406, concat("number overflow parsing '", m_lexer.get_token_string(), '\''), nullptr)); + if (!overflow_error(sax, res, allow_recovery)) + { + return false; + } + break; } if (JSON_HEDLEY_UNLIKELY(!sax->number_float(res, m_lexer.get_string()))) @@ -369,23 +418,63 @@ class parser case token_type::parse_error: { // using "uninitialized" to avoid an "expected" message - return sax->parse_error(m_lexer.get_position(), - m_lexer.get_token_string(), - parse_error::create(101, m_lexer.get_position(), exception_message(token_type::uninitialized, "value"), nullptr)); + if (!report_error(sax, parse_error::create(101, m_lexer.get_position(), exception_message(token_type::uninitialized, "value"), nullptr), allow_recovery)) + { + return false; + } + + // recover: keep what can be read of the token + recover_token(); + if (last_token != token_type::uninitialized) + { + // a string or a number + continue; + } + if (states.empty()) + { + // look for the value after the garbage + if (!skip_to_value()) + { + return false; + } + continue; + } + // nothing could be read + if (JSON_HEDLEY_UNLIKELY(!sax->null())) + { + return false; + } + break; } case token_type::end_of_input: { if (JSON_HEDLEY_UNLIKELY(m_lexer.get_position().chars_read_total == 1)) { - return sax->parse_error(m_lexer.get_position(), - m_lexer.get_token_string(), - parse_error::create(101, m_lexer.get_position(), - "attempting to parse an empty input; check that your input string or stream contains the expected JSON", nullptr)); + // there is nothing to recover + static_cast(report_error(sax, parse_error::create(101, m_lexer.get_position(), + "attempting to parse an empty input; check that your input string or stream contains the expected JSON", nullptr), allow_recovery)); + return false; } - return sax->parse_error(m_lexer.get_position(), - m_lexer.get_token_string(), - parse_error::create(101, m_lexer.get_position(), exception_message(token_type::literal_or_value, "value"), nullptr)); + if (!report_error(sax, parse_error::create(101, m_lexer.get_position(), exception_message(token_type::literal_or_value, "value"), nullptr), allow_recovery)) + { + return false; + } + + // recover: the input ends where a value is missing + if (states.empty()) + { + // there is no value + return false; + } + if (!recover_missing_value(sax, states)) + { + return false; + } + // the state evaluation reads the token again + m_lexer.unget_token(); + skip_to_state_evaluation = true; + continue; } case token_type::uninitialized: case token_type::end_array: @@ -395,9 +484,35 @@ class parser case token_type::literal_or_value: default: // the last token was unexpected { - return sax->parse_error(m_lexer.get_position(), - m_lexer.get_token_string(), - parse_error::create(101, m_lexer.get_position(), exception_message(token_type::literal_or_value, "value"), nullptr)); + if (!report_error(sax, parse_error::create(101, m_lexer.get_position(), exception_message(token_type::literal_or_value, "value"), nullptr), allow_recovery)) + { + return false; + } + + // recover + if (states.empty()) + { + // look for the value after the garbage + if (!skip_to_value()) + { + return false; + } + continue; + } + if (last_token == token_type::name_separator) + { + // a stray ':'; the value may follow + get_token(); + continue; + } + if (!recover_missing_value(sax, states)) + { + return false; + } + // the state evaluation reads the token again + m_lexer.unget_token(); + skip_to_state_evaluation = true; + continue; } } } @@ -447,9 +562,30 @@ class parser continue; } - return sax->parse_error(m_lexer.get_position(), - m_lexer.get_token_string(), - parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_array, "array"), nullptr)); + if (!report_error(sax, parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_array, "array"), nullptr), allow_recovery)) + { + return false; + } + + // recover + if (last_token == token_type::end_of_input) + { + // the input ends inside the array + return close_containers(sax, states); + } + if (last_token == token_type::end_object) + { + // a wrong closing bracket closes the innermost container + if (JSON_HEDLEY_UNLIKELY(!sax->end_array())) + { + return false; + } + states.pop_back(); + skip_to_state_evaluation = true; + } + // otherwise, a missing ',' (or a stray ':', which value + // parsing drops): the next value begins here + continue; } // states.back() is false -> object @@ -466,11 +602,12 @@ class parser // parse key if (JSON_HEDLEY_UNLIKELY(last_token != token_type::value_string)) { - return sax->parse_error(m_lexer.get_position(), - m_lexer.get_token_string(), - parse_error::create(101, m_lexer.get_position(), exception_message(token_type::value_string, "object key"), nullptr)); + if (!continue_after(key_error(sax, allow_recovery, false), skip_to_state_evaluation)) + { + return false; + } + continue; } - if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string()))) { return false; @@ -479,9 +616,11 @@ class parser // parse separator (:) if (JSON_HEDLEY_UNLIKELY(get_token() != token_type::name_separator)) { - return sax->parse_error(m_lexer.get_position(), - m_lexer.get_token_string(), - parse_error::create(101, m_lexer.get_position(), exception_message(token_type::name_separator, "object separator"), nullptr)); + if (!continue_after(key_error(sax, allow_recovery, true), skip_to_state_evaluation)) + { + return false; + } + continue; } // parse values @@ -508,12 +647,479 @@ class parser continue; } - return sax->parse_error(m_lexer.get_position(), - m_lexer.get_token_string(), - parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_object, "object"), nullptr)); + if (!report_error(sax, parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_object, "object"), nullptr), allow_recovery)) + { + return false; + } + + // recover + if (last_token == token_type::end_of_input) + { + // the input ends inside the object + return close_containers(sax, states); + } + if (last_token == token_type::end_array) + { + // a wrong closing bracket closes the innermost container + if (JSON_HEDLEY_UNLIKELY(!sax->end_object())) + { + return false; + } + states.pop_back(); + skip_to_state_evaluation = true; + continue; + } + if (!continue_after(recover_member(sax, allow_recovery), skip_to_state_evaluation)) + { + return false; + } } } + /*! + @brief continue sax_parse_internal() after a recovery + @return whether to continue parsing + */ + bool continue_after(const next_step step, bool& skip_to_state_evaluation) + { + if (step == next_step::evaluate_state) + { + // the state evaluation reads the token again + m_lexer.unget_token(); + skip_to_state_evaluation = true; + } + return step != next_step::stop; + } + + /// the parser for parse() and accept() never recovers: stop parsing + static std::false_type continue_after(std::false_type /*step*/, bool& /*skip_to_state_evaluation*/) noexcept + { + return {}; + } + + /*! + @brief parse an object key and the name separator (:) after it + + last_token is the token where the key is expected. sax_parse_internal() + repeats these steps rather than calling this function, which is used + when recovering from an error. + + @return next_step::parse_value if the value follows, with last_token its + first token; next_step::evaluate_state if the object's state is + to be evaluated after recovering from an error; next_step::stop + to stop parsing + */ + template + next_step parse_key(SAX* sax) + { + const std::true_type allow_recovery{}; + + if (JSON_HEDLEY_UNLIKELY(last_token != token_type::value_string)) + { + return key_error(sax, allow_recovery, false); + } + + if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string()))) + { + return next_step::stop; + } + + // parse separator (:) + if (JSON_HEDLEY_UNLIKELY(get_token() != token_type::name_separator)) + { + return key_error(sax, allow_recovery, true); + } + + // the value begins with the next token + get_token(); + return next_step::parse_value; + } + + /*! + @brief report a number that is too large for number_float_t, and recover + from the error by passing the value on; the SAX parser gets the + number's text as well + + This is a separate function, as reading other numbers is measurably + slower if the error is handled where they are read. + + @param[in] sax the SAX parser + @param[in] value the value that is not finite + @return whether to continue parsing + */ + template + bool overflow_error(SAX* sax, const number_float_t value, AllowRecovery allow_recovery) + { + if (!report_error(sax, out_of_range::create(406, concat("number overflow parsing '", m_lexer.get_token_string(), '\''), nullptr), allow_recovery)) + { + return false; + } + return sax->number_float(value, m_lexer.get_string()); + } + + /*! + @brief report a missing key, or a missing name separator (:) after the + key; the parser for parse() and accept() never recovers + + @param[in] key_read whether the key was read, so that the name separator + is missing + @return std::false_type, see report_error() + */ + template + std::false_type key_error(SAX* sax, std::false_type allow_recovery, const bool key_read) + { + return report_error(sax, parse_error::create(101, m_lexer.get_position(), key_read + ? exception_message(token_type::name_separator, "object separator") + : exception_message(token_type::value_string, "object key"), nullptr), allow_recovery); + } + + /*! + @brief report a missing key, or a missing name separator (:) after the + key, and recover from it + + @param[in] key_read whether the key was read, so that the name separator + is missing + */ + template + next_step key_error(SAX* sax, std::true_type allow_recovery, const bool key_read) + { + if (!key_read) + { + if (!report_error(sax, parse_error::create(101, m_lexer.get_position(), exception_message(token_type::value_string, "object key"), nullptr), allow_recovery)) + { + return next_step::stop; + } + return recover_key(sax); + } + + if (!report_error(sax, parse_error::create(101, m_lexer.get_position(), exception_message(token_type::name_separator, "object separator"), nullptr), allow_recovery)) + { + return next_step::stop; + } + return recover_name_separator(sax); + } + + ///////////////////// + // error recovery + ///////////////////// + + /* + The functions below repair an error after the SAX parser's parse_error() + returned true (see #3989). Each mistake is repaired by the smallest local + edit: a missing ',' or ':' is inserted, a stray token is removed, what can + be read of an invalid string or number is kept (see + lexer::recover_token()), a missing value becomes null, a wrong closing + bracket closes the innermost container, and the end of the input closes + all of them. The events stay balanced, and every key() is followed by + exactly one value. + + A repair hands a token to the state evaluation, by returning it to the + lexer (lexer::unget_token()) so that the state evaluation reads it again, + only if it is ',', ']', '}', or the end of the input. The state evaluation + hands a token to value or key parsing only if it is none of them, so a + token is never handed back and forth. Every other step reads a token or + closes a container, so parsing always ends. + */ + + /*! + @brief report an error to the SAX parser; the parser for parse() and + accept() never recovers + + @return std::false_type rather than false: its value is known where the + function is called even if the call is not inlined, so the code + for recovering is not generated + */ + template + std::false_type report_error(SAX* sax, const Exception& ex, std::false_type /*allow_recovery*/) + { + error_reported = true; + static_cast(sax->parse_error(m_lexer.get_position(), m_lexer.get_token_string(), ex)); + return {}; + } + + /*! + @brief report an error to the SAX parser + @return whether to recover from the error + */ + template + bool report_error(SAX* sax, const Exception& ex, std::true_type /*allow_recovery*/) + { + const std::size_t position = m_lexer.get_position().chars_read_total; + if (error_reported && position == last_error_position && last_token == last_error_token) + { + // a repair handed on the token of the error it repaired; the + // token was reported already, and the SAX parser asked to recover + return true; + } + + error_reported = true; + last_error_position = position; + last_error_token = last_token; + + if (!sax->parse_error(m_lexer.get_position(), m_lexer.get_token_string(), ex)) + { + return false; + } + + // the token string of the next error begins here + m_lexer.restart_token_string(); + return true; + } + + /*! + @brief keep what can be read of the token that the lexer rejected + + The error was reported for the rejected token, so it is not reported again + for the token it is repaired to (see lexer::recover_token()). + */ + token_type recover_token() + { + last_token = m_lexer.recover_token(); + last_error_position = m_lexer.get_position().chars_read_total; + last_error_token = last_token; + return last_token; + } + + /// pass the end events of all open containers + template + bool close_containers(SAX* sax, std::vector& states) + { + while (!states.empty()) + { + const bool is_array = states.back(); + states.pop_back(); + if (JSON_HEDLEY_UNLIKELY(is_array ? !sax->end_array() : !sax->end_object())) + { + return false; + } + } + return true; + } + + /*! + @brief read tokens until one begins a value, skipping everything before + the top-level value + @return whether a value begins with last_token + */ + bool skip_to_value() + { + while (true) + { + switch (get_token()) + { + case token_type::begin_array: + case token_type::begin_object: + case token_type::literal_false: + case token_type::literal_null: + case token_type::literal_true: + case token_type::value_float: + case token_type::value_integer: + case token_type::value_string: + case token_type::value_unsigned: + return true; + + case token_type::end_of_input: + return false; + + case token_type::parse_error: + recover_token(); + if (last_token != token_type::uninitialized) + { + return true; + } + break; + + case token_type::uninitialized: + case token_type::end_array: + case token_type::end_object: + case token_type::name_separator: + case token_type::value_separator: + case token_type::literal_or_value: + default: + break; + } + } + } + + /*! + @brief skip the rest of an object member that cannot be read + + Reads tokens, beginning with last_token, until a ',', '}', or ']' that is + not inside a container that begins in the skipped tokens, or the end of + the input. + */ + void skip_member() + { + std::size_t depth = 0; + while (true) + { + switch (last_token) + { + case token_type::begin_array: + case token_type::begin_object: + ++depth; + break; + + case token_type::end_array: + case token_type::end_object: + if (depth == 0) + { + return; + } + --depth; + break; + + case token_type::value_separator: + if (depth == 0) + { + return; + } + break; + + case token_type::end_of_input: + return; + + case token_type::parse_error: + recover_token(); + break; + + case token_type::uninitialized: + case token_type::literal_true: + case token_type::literal_false: + case token_type::literal_null: + case token_type::value_string: + case token_type::value_unsigned: + case token_type::value_integer: + case token_type::value_float: + case token_type::name_separator: + case token_type::literal_or_value: + default: + break; + } + get_token(); + } + } + + /*! + @brief pass a value where it is missing + + last_token is ',', ']', '}', or the end of the input, where a value was + expected. In an object, the key gets null; in an array, a ',' where a + value is missing stands for null (as in JavaScript), while an array that + ends there just ends. + */ + template + bool recover_missing_value(SAX* sax, const std::vector& states) + { + JSON_ASSERT(!states.empty()); + if (!states.back() || last_token == token_type::value_separator) + { + return sax->null(); + } + return true; + } + + /// recover from a missing key; last_token is where it was expected + template + next_step recover_key(SAX* sax) + { + switch (last_token) + { + case token_type::value_separator: + case token_type::end_object: + case token_type::end_array: + case token_type::end_of_input: + // no member: the object's state handles the token + return next_step::evaluate_state; + + case token_type::parse_error: + recover_token(); + if (last_token == token_type::value_string) + { + // a key that could be repaired + return parse_key(sax); + } + skip_member(); + return next_step::evaluate_state; + + case token_type::uninitialized: + case token_type::literal_true: + case token_type::literal_false: + case token_type::literal_null: + case token_type::value_string: + case token_type::value_unsigned: + case token_type::value_integer: + case token_type::value_float: + case token_type::begin_array: + case token_type::begin_object: + case token_type::name_separator: + case token_type::literal_or_value: + default: + // a member without a key + skip_member(); + return next_step::evaluate_state; + } + } + + /// recover from a missing name separator (:) after the key; last_token + /// is where it was expected + template + next_step recover_name_separator(SAX* sax) + { + switch (last_token) + { + case token_type::value_separator: + case token_type::end_object: + case token_type::end_array: + case token_type::end_of_input: + // the value is missing as well + return sax->null() ? next_step::evaluate_state : next_step::stop; + + case token_type::uninitialized: + case token_type::literal_true: + case token_type::literal_false: + case token_type::literal_null: + case token_type::value_string: + case token_type::value_unsigned: + case token_type::value_integer: + case token_type::value_float: + case token_type::begin_array: + case token_type::begin_object: + case token_type::name_separator: + case token_type::parse_error: + case token_type::literal_or_value: + default: + // a missing ':'; the value begins here + return next_step::parse_value; + } + } + + /// recover from a token after an object member that is neither ',' nor + /// '}' (nor ']' or the end of the input, which the caller handles) + template + next_step recover_member(SAX* sax, std::true_type /*allow_recovery*/) + { + if (last_token == token_type::parse_error) + { + recover_token(); + } + if (last_token == token_type::value_string) + { + // a missing ','; the next key begins here + return parse_key(sax); + } + skip_member(); + return next_step::evaluate_state; + } + + /// the parser for parse() and accept() never recovers (and does not come + /// here, as report_error() returned false) + template + std::false_type recover_member(SAX* /*sax*/, std::false_type /*allow_recovery*/) const noexcept + { + return {}; + } + /// get next token from lexer token_type get_token() { @@ -560,6 +1166,12 @@ class parser const bool allow_exceptions = true; /// whether trailing commas in objects and arrays should be ignored (true) or signaled as errors (false) const bool ignore_trailing_commas = false; + /// whether an error was reported to the SAX parser + bool error_reported = false; + /// the position of the last reported error + std::size_t last_error_position = 0; + /// the token of the last reported error + token_type last_error_token = token_type::uninitialized; }; } // namespace detail diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 500fcddf2..c144a87df 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -4979,6 +4979,26 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec return parser(i.get(), nullptr, false, ignore_comments, ignore_trailing_commas, true).accept(true); } + private: + /// read a binary format and pass it to a SAX parser; if the SAX parser + /// asks to recover from an error, the value read so far is completed + /// (see detail::json_sax_salvager and #3989) + template + static bool sax_parse_binary(InputAdapterType ia, SAX* sax, + const input_format_t format, const bool strict) + { + (void)detail::is_sax_static_asserts {}; + using salvager_t = detail::json_sax_salvager; + salvager_t salvager(sax); + const bool result = detail::binary_reader(std::move(ia), format).sax_parse(format, &salvager, strict); + if (!result) + { + salvager.close_open_containers(); + } + return result; + } + + public: /// @brief generate SAX events /// @sa https://json.nlohmann.me/api/basic_json/sax_parse/ template @@ -4992,7 +5012,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec auto ia = detail::input_adapter(std::forward(i)); return format == input_format_t::json ? parser(std::move(ia), nullptr, true, ignore_comments, ignore_trailing_commas).sax_parse(sax, strict) - : detail::binary_reader(std::move(ia), format).sax_parse(format, sax, strict); + : sax_parse_binary(std::move(ia), sax, format, strict); } /// @brief generate SAX events (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -5009,7 +5029,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec auto ia = detail::input_adapter(std::move(first), std::move(last)); return format == input_format_t::json ? parser(std::move(ia), nullptr, true, ignore_comments, ignore_trailing_commas).sax_parse(sax, strict) - : detail::binary_reader(std::move(ia), format).sax_parse(format, sax, strict); + : sax_parse_binary(std::move(ia), sax, format, strict); } /// @brief generate SAX events @@ -5031,7 +5051,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) ? parser(std::move(ia), nullptr, true, ignore_comments, ignore_trailing_commas).sax_parse(sax, strict) // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) - : detail::binary_reader(std::move(ia), format).sax_parse(format, sax, strict); + : sax_parse_binary(std::move(ia), sax, format, strict); } #ifndef JSON_NO_IO /// @brief deserialize from stream diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 0e3cae486..0ab2db096 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -8474,6 +8474,7 @@ NLOHMANN_JSON_NAMESPACE_END #include // array #include // localeconv #include // size_t +#include // uint8_t #include // snprintf #include // strtof, strtod, strtold, strtoll, strtoull #include // initializer_list @@ -9550,8 +9551,16 @@ class lexer : public lexer_base if (0xD800 <= codepoint1 && codepoint1 <= 0xDBFF) { // expect next \uxxxx entry - if (JSON_HEDLEY_LIKELY(get() == '\\' && get() == 'u')) + if (JSON_HEDLEY_LIKELY(get() == '\\')) { + if (JSON_HEDLEY_UNLIKELY(get() != 'u')) + { + // current is the character escaped by the backslash + error_message = "invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF"; + string_error_resume = resume_kind::escaped_character; + return token_type::parse_error; + } + const int codepoint2 = get_codepoint(); if (JSON_HEDLEY_UNLIKELY(codepoint2 == -1)) @@ -9576,7 +9585,11 @@ class lexer : public lexer_base } else { + // the second escape was read completely and is a + // code point of its own error_message = "invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF"; + string_error_resume = resume_kind::after_escape; + string_error_codepoint = codepoint2; return token_type::parse_error; } } @@ -9590,7 +9603,9 @@ class lexer : public lexer_base { if (JSON_HEDLEY_UNLIKELY(0xDC00 <= codepoint1 && codepoint1 <= 0xDFFF)) { + // the escape was read completely error_message = "invalid string: surrogate U+DC00..U+DFFF must follow U+D800..U+DBFF"; + string_error_resume = resume_kind::after_escape; return token_type::parse_error; } } @@ -11335,6 +11350,552 @@ scan_number_done: } } + ///////////////////// + // error recovery + ///////////////////// + + /*! + @brief make the best of the token that scan() rejected + + Called by the parser after scan() returned token_type::parse_error and the + SAX parser asked to recover from the error (see #3989). Keeps what can be + read of the token and skips the rest: + + - A string keeps its characters. An unknown escape stands for the escaped + character itself (as in JavaScript), an invalid `\u` escape and ill-formed + UTF-8 become U+FFFD, and a control character is kept. A line break or the + end of the input ends a string that lacks its closing quote. + - A number keeps its longest valid prefix, e.g. `1` for `1.` or `1e+`. + - A block comment that is not closed runs to the end of the input. + - Anything else is skipped. + + The rest of an invalid token is skipped up to the next delimiter + (whitespace, a structural character, or a quote). A delimiter that the + invalid token consumed is returned to the input, so that the next scan() + reads it. + + @return token_type::value_string or a number token type if a string or a + number could be read, token_type::end_of_input for a block comment + that is not closed, token_type::uninitialized otherwise + */ + token_type recover_token() + { + const resume_kind resume = string_error_resume; + const int codepoint = string_error_codepoint; + string_error_resume = resume_kind::character; + string_error_codepoint = -1; + + if (error_message_starts_with("invalid string")) + { + return recover_string(resume, codepoint); + } + + if (error_message_starts_with("invalid number")) + { + return recover_number(); + } + + if (error_message_starts_with("invalid comment; missing")) + { + // the comment runs to the end of the input + return token_type::end_of_input; + } + + skip_to_delimiter(); + return token_type::uninitialized; + } + + /*! + @brief return the token that scan() read last to the input, so that the + next scan() reads it again + + Called by the parser when recovering from an error. The token must be a + single character (',', ':', '[', ']', '{', or '}') or the end of the + input, and scan() must have read it last. + */ + void unget_token() + { + JSON_ASSERT(!next_unget); + unget(); + } + + /*! + @brief let the token string for the next error begin at the current character + + The token string of an error reaches back to the beginning of the last + string or number. After an error, the parser calls this function so that + the next error does not report (and, with many errors, copy) everything + read since then. + */ + void restart_token_string() + { + restart_token_string_impl(std::integral_constant {}); + } + + private: + /// how recover_string() continues after the error scan_string() reported + enum class resume_kind : std::uint8_t + { + /// current is the next character of the string (or the end of input) + character, + /// current is the character escaped by the preceding backslash + escaped_character, + /// current is the last character of a complete escape + after_escape + }; + + /// whether error_message begins with @a prefix + bool error_message_starts_with(const char* prefix) const noexcept + { + const char* message = error_message; + while (*prefix != '\0') + { + if (*message++ != *prefix++) + { + return false; + } + } + return true; + } + + /// whether current ends an invalid token (see recover_token()) + bool current_is_delimiter() const noexcept + { + switch (current) + { + case ' ': + case '\t': + case '\n': + case '\r': + case '[': + case ']': + case '{': + case '}': + case ',': + case ':': + case '\"': +#if !JSON_STRICT_NUL_HANDLING + case '\0': +#endif + case char_traits::eof(): + return true; + + case '/': + return ignore_comments; + + default: + return false; + } + } + + /// skip the rest of an invalid token and return its delimiter to the input + void skip_to_delimiter() + { + while (!current_is_delimiter()) + { + get(); + } + + if (current != char_traits::eof()) + { + unget(); + } + } + + /// append U+FFFD REPLACEMENT CHARACTER to token_buffer + void add_replacement_character() + { + add(0xEF); + add(0xBF); + add(0xBD); + } + + /// append the UTF-8 encoding of @a codepoint (not a surrogate) to token_buffer + void add_codepoint(const int codepoint) + { + JSON_ASSERT(0x00 <= codepoint && codepoint <= 0x10FFFF); + const auto cp = static_cast(codepoint); + if (cp < 0x80) + { + add(static_cast(cp)); + } + else if (cp <= 0x7FF) + { + add(static_cast(0xC0u | (cp >> 6u))); + add(static_cast(0x80u | (cp & 0x3Fu))); + } + else if (cp <= 0xFFFF) + { + add(static_cast(0xE0u | (cp >> 12u))); + add(static_cast(0x80u | ((cp >> 6u) & 0x3Fu))); + add(static_cast(0x80u | (cp & 0x3Fu))); + } + else + { + add(static_cast(0xF0u | (cp >> 18u))); + add(static_cast(0x80u | ((cp >> 12u) & 0x3Fu))); + add(static_cast(0x80u | ((cp >> 6u) & 0x3Fu))); + add(static_cast(0x80u | (cp & 0x3Fu))); + } + } + + /// append a code point read from a `\u` escape; a surrogate becomes U+FFFD + void add_escaped_codepoint(const int codepoint) + { + if (0xD800 <= codepoint && codepoint <= 0xDFFF) + { + add_replacement_character(); + } + else + { + add_codepoint(codepoint); + } + } + + /*! + @brief remove an incomplete UTF-8 sequence from the end of token_buffer + + next_byte_in_range() adds the bytes of a sequence as it checks them, so + when it rejects a byte, the beginning of the sequence is already in + token_buffer, which otherwise holds only complete sequences. + + @return whether an incomplete sequence was removed + */ + bool remove_incomplete_utf8_sequence() + { + std::size_t lead = token_buffer.size(); + std::size_t continuation_bytes = 0; + while (lead > 0 && continuation_bytes < 3 + && (static_cast(token_buffer[lead - 1]) & 0xC0u) == 0x80u) + { + --lead; + ++continuation_bytes; + } + if (lead == 0) + { + return false; + } + + const auto lead_byte = static_cast(token_buffer[lead - 1]); + const std::size_t expected = (lead_byte >= 0xF0) ? 3 : (lead_byte >= 0xE0) ? 2 : (lead_byte >= 0xC0) ? 1 : 0; + if (continuation_bytes >= expected) + { + return false; + } + + token_buffer.resize(lead - 1); + return true; + } + + /*! + @brief read the UTF-8 sequence that begins with current, which is not ASCII + @return whether the next character must be read; false if current still + needs to be handled, because it does not belong to the sequence + */ + bool recover_utf8_sequence() + { + // the number of continuation bytes and the range of the first one; + // see the ranges in scan_string() + std::size_t count = 0; + char_int_type low = 0x80; + char_int_type high = 0xBF; + if (current >= 0xC2 && current <= 0xDF) + { + count = 1; + } + else if (current >= 0xE0 && current <= 0xEF) + { + count = 2; + low = (current == 0xE0) ? 0xA0 : 0x80; + high = (current == 0xED) ? 0x9F : 0xBF; + } + else if (current >= 0xF0 && current <= 0xF4) + { + count = 3; + low = (current == 0xF0) ? 0x90 : 0x80; + high = (current == 0xF4) ? 0x8F : 0xBF; + } + else + { + // an ill-formed byte + add_replacement_character(); + return true; + } + + const std::size_t start = token_buffer.size(); + add(current); + for (std::size_t i = 0; i < count; ++i) + { + get(); + if (current < low || current > high) + { + token_buffer.resize(start); + add_replacement_character(); + return false; + } + add(current); + low = 0x80; + high = 0xBF; + } + return true; + } + + /*! + @brief read the low surrogate that must follow the high surrogate @a high + @return whether the next character must be read; false if current still + needs to be handled + */ + bool recover_low_surrogate(int high) + { + while (true) + { + if (get() != '\\') + { + add_replacement_character(); + return false; + } + if (get() != 'u') + { + add_replacement_character(); + // not 'u', so this does not come back here + return recover_escape(); + } + + const int low = get_codepoint(); + if (low == -1) + { + add_replacement_character(); + return false; + } + if (0xDC00 <= low && low <= 0xDFFF) + { + add_codepoint(static_cast((static_cast(high) << 10u) + + static_cast(low) - 0x35FDC00u)); + return true; + } + + // high has no low surrogate + add_replacement_character(); + if (low < 0xD800 || low > 0xDBFF) + { + add_codepoint(low); + return true; + } + // another high surrogate + high = low; + } + } + + /*! + @brief read the escape whose backslash was read; current is the escaped character + @return whether the next character must be read; false if current still + needs to be handled + */ + bool recover_escape() + { + switch (current) + { + case '\"': + add('\"'); + return true; + case '\\': + add('\\'); + return true; + case '/': + add('/'); + return true; + case 'b': + add('\b'); + return true; + case 'f': + add('\f'); + return true; + case 'n': + add('\n'); + return true; + case 'r': + add('\r'); + return true; + case 't': + add('\t'); + return true; + + case 'u': + { + const int codepoint = get_codepoint(); + if (codepoint == -1) + { + add_replacement_character(); + return false; + } + if (0xD800 <= codepoint && codepoint <= 0xDBFF) + { + return recover_low_surrogate(codepoint); + } + add_escaped_codepoint(codepoint); + return true; + } + + // an unknown escape stands for the escaped character + default: + return false; + } + } + + /*! + @brief read the rest of a string after scan_string() rejected it + + token_buffer holds what scan_string() read before the error. See + recover_token() for how errors are repaired. + + @param[in] resume how to continue, see resume_kind + @param[in] codepoint for a high surrogate followed by an escape of another + code point: that code point; -1 otherwise + */ + token_type recover_string(const resume_kind resume, const int codepoint) + { + // whether the next character must be read before it can be handled + bool fetch = false; + + if (error_message_starts_with("invalid string: surrogate") + || error_message_starts_with("invalid string: '\\u'")) + { + add_replacement_character(); + } + else if (error_message_starts_with("invalid string: ill-formed UTF-8") + && remove_incomplete_utf8_sequence()) + { + add_replacement_character(); + } + + switch (resume) + { + case resume_kind::escaped_character: + fetch = recover_escape(); + break; + case resume_kind::after_escape: + if (0xD800 <= codepoint && codepoint <= 0xDBFF) + { + fetch = recover_low_surrogate(codepoint); + } + else + { + if (codepoint != -1) + { + add_escaped_codepoint(codepoint); + } + fetch = true; + } + break; + case resume_kind::character: + default: + break; + } + + while (true) + { + if (fetch) + { + get(); + } + fetch = true; + + switch (current) + { + case '\"': + // a line break or the end of the input ends a string that + // lacks its closing quote + case '\n': + case '\r': + case char_traits::eof(): + return token_type::value_string; + +#if !JSON_STRICT_NUL_HANDLING + case '\0': + // the end of the input, see scan() + unget(); + return token_type::value_string; +#endif + + case '\\': + get(); + fetch = recover_escape(); + break; + + default: + if (current < 0x80) + { + // including control characters + add(current); + } + else + { + fetch = recover_utf8_sequence(); + } + break; + } + } + } + + /*! + @brief keep the longest valid prefix of a number that scan_number() rejected + + token_buffer holds the characters scan_number() accepted before the error, + so the prefix ends at its last digit. + */ + token_type recover_number() + { + while (!token_buffer.empty() && (token_buffer.back() < '0' || token_buffer.back() > '9')) + { + token_buffer.pop_back(); + } + + if (token_buffer.empty()) + { + skip_to_delimiter(); + return token_type::uninitialized; + } + + if (decimal_point_position >= token_buffer.size()) + { + decimal_point_position = std::string::npos; + } + + const std::size_t exponent = token_buffer.find_first_of("eE"); + const std::size_t mantissa_end = (exponent == std::string::npos) ? token_buffer.size() : exponent; + token_type number_type = token_type::value_unsigned; + if (decimal_point_position != std::string::npos || exponent != std::string::npos) + { + number_type = token_type::value_float; + } + else if (token_buffer.front() == '-') + { + number_type = token_type::value_integer; + } + + const token_type result = convert_number(number_type, mantissa_end); + skip_to_delimiter(); + return result; + } + + /// seekable adapter: the token string begins at current, which was consumed + void restart_token_string_impl(std::true_type /*lazy*/) noexcept + { + const std::size_t consumed = ia.get_consumed_count(); + token_string_start = (consumed > 0 && current != char_traits::eof()) ? consumed - 1 : consumed; + } + + /// streaming adapter: the token string begins at current; a character + /// that was put back is copied again when it is read again + void restart_token_string_impl(std::false_type /*lazy*/) + { + token_string.clear(); + if (!next_unget && current != char_traits::eof()) + { + token_string.push_back(char_traits::to_char_type(current)); + } + } + private: /// input adapter InputAdapterType ia; @@ -11375,6 +11936,13 @@ scan_number_done: /// a description of occurred lexer errors const char* error_message = ""; + /// how recover_token() continues a string that scan_string() rejected; + /// set only on the error paths that need more than error_message + resume_kind string_error_resume = resume_kind::character; + /// the code point of the second escape when a high surrogate is followed + /// by an escape that is not a low surrogate; -1 otherwise + int string_error_codepoint = -1; + // number values number_integer_t value_integer = 0; number_unsigned_t value_unsigned = 0; @@ -11513,7 +12081,9 @@ struct json_sax @param[in] position the position in the input where the error occurs @param[in] last_token the last read token @param[in] ex an exception object describing the error - @return whether parsing should proceed (must return false) + @return whether to recover from the error: false stops parsing; true + repairs JSON text and continues, or, for the binary formats, stops + after closing the containers read so far */ virtual bool parse_error(std::size_t position, const std::string& last_token, @@ -11568,9 +12138,12 @@ a pointer to the respective array or object for each recursion depth. After successful parsing, the value that is passed by reference to the constructor contains the parsed value. -@tparam BasicJsonType the JSON type +@tparam BasicJsonType the JSON type +@tparam InputAdapterType the input adapter of the lexer that can be passed to + the constructor to record diagnostic positions; it + does not matter if no lexer is passed */ -template +template class json_sax_dom_parser { public: @@ -11887,7 +12460,7 @@ class json_sax_dom_parser lexer_t* m_lexer_ref = nullptr; }; -template +template class json_sax_dom_callback_parser { public: @@ -12589,6 +13162,177 @@ class json_sax_acceptor } }; +/*! +@brief SAX proxy that lets the binary readers keep what was read before an error + +The binary formats cannot continue after an error: a value's size is given +before its payload, and every byte value is a valid type marker, so there is no +way to find where the next value begins. When the SAX parser's parse_error() +returns true to ask for error recovery, the best the binary readers can offer is +the value read up to the error. + +This proxy forwards every event to the SAX parser and records which containers +are open and whether a key still waits for its value. After an error the SAX +parser asked to recover from, @ref close_open_containers then completes the +value with null for a pending key and the missing end events, so the SAX parser +sees balanced events (see #3989). + +@tparam BasicJsonType the JSON type +@tparam SAX the SAX parser to forward the events to +*/ +template +class json_sax_salvager +{ + public: + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using binary_t = typename BasicJsonType::binary_t; + + explicit json_sax_salvager(SAX* sax_) noexcept + : sax(sax_) + {} + + bool null() + { + key_pending = false; + return sax->null(); + } + + bool boolean(bool val) + { + key_pending = false; + return sax->boolean(val); + } + + bool number_integer(number_integer_t val) + { + key_pending = false; + return sax->number_integer(val); + } + + bool number_unsigned(number_unsigned_t val) + { + key_pending = false; + return sax->number_unsigned(val); + } + + bool number_float(number_float_t val, const string_t& s) + { + key_pending = false; + return sax->number_float(val, s); + } + + bool string(string_t& val) + { + key_pending = false; + return sax->string(val); + } + + bool binary(binary_t& val) + { + key_pending = false; + return sax->binary(val); + } + + bool start_object(std::size_t len) + { + key_pending = false; + if (JSON_HEDLEY_UNLIKELY(!sax->start_object(len))) + { + return false; + } + open_containers.push_back(true); + return true; + } + + bool key(string_t& val) + { + key_pending = true; + return sax->key(val); + } + + bool end_object() + { + JSON_ASSERT(!open_containers.empty() && open_containers.back()); + open_containers.pop_back(); + return sax->end_object(); + } + + bool start_array(std::size_t len) + { + key_pending = false; + if (JSON_HEDLEY_UNLIKELY(!sax->start_array(len))) + { + return false; + } + open_containers.push_back(false); + return true; + } + + bool end_array() + { + JSON_ASSERT(!open_containers.empty() && !open_containers.back()); + open_containers.pop_back(); + return sax->end_array(); + } + + template + bool parse_error(std::size_t position, const std::string& last_token, + const Exception& ex) + { + recovery_requested = sax->parse_error(position, last_token, ex); + // the binary readers stop after an error anyway + return false; + } + + /*! + @brief complete the value read before an error + + Does nothing unless the SAX parser's parse_error() returned true. Otherwise + passes null for a key that waits for its value and closes the containers + that are still open, innermost first, until an event returns false. + */ + void close_open_containers() + { + if (!recovery_requested) + { + return; + } + recovery_requested = false; + + if (key_pending) + { + key_pending = false; + if (JSON_HEDLEY_UNLIKELY(!sax->null())) + { + return; + } + } + + while (!open_containers.empty()) + { + const bool is_object = open_containers.back(); + open_containers.pop_back(); + if (JSON_HEDLEY_UNLIKELY(is_object ? !sax->end_object() : !sax->end_array())) + { + return; + } + } + } + + private: + /// the SAX parser the events are forwarded to + SAX* sax = nullptr; + /// the containers that are open, innermost last; true for an object + std::vector open_containers {}; // NOLINT(readability-redundant-member-init) + /// whether a key was passed whose value has not been passed yet + bool key_pending = false; + /// whether the SAX parser's parse_error() asked to recover from the error + bool recovery_requested = false; +}; + } // namespace detail NLOHMANN_JSON_NAMESPACE_END @@ -12923,8 +13667,8 @@ class binary_reader if (JSON_HEDLEY_UNLIKELY(current != char_traits::eof())) { - return sax->parse_error(chars_read, get_token_string(), parse_error::create(110, chars_read, - exception_message(input_format, concat("expected end of input; last byte: 0x", get_token_string()), "value"), nullptr)); + return report_error(chars_read, get_token_string(), parse_error::create(110, chars_read, + exception_message(input_format, concat("expected end of input; last byte: 0x", get_token_string()), "value"), nullptr)); } } @@ -13030,8 +13774,8 @@ class binary_reader { if (JSON_HEDLEY_UNLIKELY(document_size < 0 || static_cast(document_size) != chars_read - document_start)) { - return sax->parse_error(chars_read, get_token_string(), parse_error::create(112, chars_read, - exception_message(input_format_t::bson, concat("document size ", std::to_string(document_size), " does not match the number of bytes read (", std::to_string(chars_read - document_start), ")"), "document"), nullptr)); + return report_error(chars_read, get_token_string(), parse_error::create(112, chars_read, + exception_message(input_format_t::bson, concat("document size ", std::to_string(document_size), " does not match the number of bytes read (", std::to_string(chars_read - document_start), ")"), "document"), nullptr)); } return true; } @@ -13225,8 +13969,8 @@ class binary_reader if (JSON_HEDLEY_UNLIKELY(len < 1)) { auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, - exception_message(input_format_t::bson, concat("string length must be at least 1, is ", std::to_string(len)), "string"), nullptr)); + return report_error(chars_read, last_token, parse_error::create(112, chars_read, + exception_message(input_format_t::bson, concat("string length must be at least 1, is ", std::to_string(len)), "string"), nullptr)); } if (JSON_HEDLEY_UNLIKELY(!get_string(input_format_t::bson, len - static_cast(1), result))) @@ -13237,10 +13981,10 @@ class binary_reader if (JSON_HEDLEY_UNLIKELY(get() != 0x00)) { auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, - exception_message(input_format_t::bson, - "BSON string is not null-terminated", - "string"), nullptr)); + return report_error(chars_read, last_token, parse_error::create(112, chars_read, + exception_message(input_format_t::bson, + "BSON string is not null-terminated", + "string"), nullptr)); } return true; @@ -13261,8 +14005,8 @@ class binary_reader if (JSON_HEDLEY_UNLIKELY(len < 0)) { auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, - exception_message(input_format_t::bson, concat("byte array length cannot be negative, is ", std::to_string(len)), "binary"), nullptr)); + return report_error(chars_read, last_token, parse_error::create(112, chars_read, + exception_message(input_format_t::bson, concat("byte array length cannot be negative, is ", std::to_string(len)), "binary"), nullptr)); } // All BSON binary values have a subtype @@ -13355,8 +14099,8 @@ class binary_reader std::array cr{{}}; static_cast((std::snprintf)(cr.data(), cr.size(), "%.2hhX", static_cast(element_type))); // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg) const std::string cr_str{cr.data()}; - return sax->parse_error(element_type_parse_position, cr_str, - parse_error::create(114, element_type_parse_position, concat("Unsupported BSON record type 0x", cr_str), nullptr)); + return report_error(element_type_parse_position, cr_str, + parse_error::create(114, element_type_parse_position, concat("Unsupported BSON record type 0x", cr_str), nullptr)); } } } @@ -13376,9 +14120,9 @@ class binary_reader const auto max_val = static_cast((std::numeric_limits::max)()); if (number > max_val) { - return sax->parse_error(chars_read, get_token_string(), - parse_error::create(112, chars_read, - exception_message(input_format_t::cbor, "negative integer overflow", "value"), nullptr)); + return report_error(chars_read, get_token_string(), + parse_error::create(112, chars_read, + exception_message(input_format_t::cbor, "negative integer overflow", "value"), nullptr)); } return sax->number_integer(static_cast(-1) - static_cast(number)); } @@ -13713,8 +14457,8 @@ class binary_reader case cbor_tag_handler_t::error: { auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, - exception_message(input_format_t::cbor, concat("invalid byte: 0x", last_token), "value"), nullptr)); + return report_error(chars_read, last_token, parse_error::create(112, chars_read, + exception_message(input_format_t::cbor, concat("invalid byte: 0x", last_token), "value"), nullptr)); } case cbor_tag_handler_t::ignore: @@ -13912,8 +14656,8 @@ class binary_reader default: // anything else (0xFF is handled inside the other types) { auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, - exception_message(input_format_t::cbor, concat("invalid byte: 0x", last_token), "value"), nullptr)); + return report_error(chars_read, last_token, parse_error::create(112, chars_read, + exception_message(input_format_t::cbor, concat("invalid byte: 0x", last_token), "value"), nullptr)); } } } @@ -13992,8 +14736,8 @@ class binary_reader default: { auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, - exception_message(input_format_t::cbor, concat("expected length specification (0x60-0x7B) or indefinite string type (0x7F); last byte: 0x", last_token), "string"), nullptr)); + return report_error(chars_read, last_token, parse_error::create(113, chars_read, + exception_message(input_format_t::cbor, concat("expected length specification (0x60-0x7B) or indefinite string type (0x7F); last byte: 0x", last_token), "string"), nullptr)); } } } @@ -14137,8 +14881,8 @@ class binary_reader default: { auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, - exception_message(input_format_t::cbor, concat("expected length specification (0x40-0x5B) or indefinite binary array type (0x5F); last byte: 0x", last_token), "binary"), nullptr)); + return report_error(chars_read, last_token, parse_error::create(113, chars_read, + exception_message(input_format_t::cbor, concat("expected length specification (0x40-0x5B) or indefinite binary array type (0x5F); last byte: 0x", last_token), "binary"), nullptr)); } } } @@ -14218,8 +14962,8 @@ class binary_reader { if (JSON_HEDLEY_UNLIKELY(!value_in_range_of(len) || len == detail::unknown_size())) { - return sax->parse_error(chars_read, get_token_string(), out_of_range::create(408, - exception_message(input_format_t::cbor, concat("excessive ", context, " size"), "size"), nullptr)); + return report_error(chars_read, get_token_string(), out_of_range::create(408, + exception_message(input_format_t::cbor, concat("excessive ", context, " size"), "size"), nullptr)); } result = conditional_static_cast(len); return true; @@ -14715,8 +15459,8 @@ class binary_reader default: // anything else { auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, - exception_message(input_format_t::msgpack, concat("invalid byte: 0x", last_token), "value"), nullptr)); + return report_error(chars_read, last_token, parse_error::create(112, chars_read, + exception_message(input_format_t::msgpack, concat("invalid byte: 0x", last_token), "value"), nullptr)); } } } @@ -14798,8 +15542,8 @@ class binary_reader default: { auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, - exception_message(input_format_t::msgpack, concat("expected length specification (0xA0-0xBF, 0xD9-0xDB); last byte: 0x", last_token), "string"), nullptr)); + return report_error(chars_read, last_token, parse_error::create(113, chars_read, + exception_message(input_format_t::msgpack, concat("expected length specification (0xA0-0xBF, 0xD9-0xDB); last byte: 0x", last_token), "string"), nullptr)); } } } @@ -15116,8 +15860,8 @@ class binary_reader { if (JSON_HEDLEY_UNLIKELY(len < 0)) { - return sax->parse_error(chars_read, get_token_string(), parse_error::create(113, chars_read, - exception_message(input_format, "string length must not be negative", "string"), nullptr)); + return report_error(chars_read, get_token_string(), parse_error::create(113, chars_read, + exception_message(input_format, "string length must not be negative", "string"), nullptr)); } return true; } @@ -15228,7 +15972,7 @@ class binary_reader { message = "expected length type specification (U, i, u, I, m, l, M, L); last byte: 0x" + last_token; } - return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format, message, "string"), nullptr)); + return report_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format, message, "string"), nullptr)); } /*! @@ -15329,8 +16073,8 @@ class binary_reader } if (number < 0) { - return sax->parse_error(chars_read, get_token_string(), parse_error::create(113, chars_read, - exception_message(input_format, "count in an optimized container must be positive", "size"), nullptr)); + return report_error(chars_read, get_token_string(), parse_error::create(113, chars_read, + exception_message(input_format, "count in an optimized container must be positive", "size"), nullptr)); } result = static_cast(number); // NOLINT(bugprone-signed-char-misuse,cert-str34-c): number is not a char return true; @@ -15345,8 +16089,8 @@ class binary_reader } if (number < 0) { - return sax->parse_error(chars_read, get_token_string(), parse_error::create(113, chars_read, - exception_message(input_format, "count in an optimized container must be positive", "size"), nullptr)); + return report_error(chars_read, get_token_string(), parse_error::create(113, chars_read, + exception_message(input_format, "count in an optimized container must be positive", "size"), nullptr)); } result = static_cast(number); return true; @@ -15361,8 +16105,8 @@ class binary_reader } if (number < 0) { - return sax->parse_error(chars_read, get_token_string(), parse_error::create(113, chars_read, - exception_message(input_format, "count in an optimized container must be positive", "size"), nullptr)); + return report_error(chars_read, get_token_string(), parse_error::create(113, chars_read, + exception_message(input_format, "count in an optimized container must be positive", "size"), nullptr)); } result = static_cast(number); return true; @@ -15377,13 +16121,13 @@ class binary_reader } if (number < 0) { - return sax->parse_error(chars_read, get_token_string(), parse_error::create(113, chars_read, - exception_message(input_format, "count in an optimized container must be positive", "size"), nullptr)); + return report_error(chars_read, get_token_string(), parse_error::create(113, chars_read, + exception_message(input_format, "count in an optimized container must be positive", "size"), nullptr)); } if (!value_in_range_of(number)) { - return sax->parse_error(chars_read, get_token_string(), out_of_range::create(408, - exception_message(input_format, "integer value overflow", "size"), nullptr)); + return report_error(chars_read, get_token_string(), out_of_range::create(408, + exception_message(input_format, "integer value overflow", "size"), nullptr)); } result = static_cast(number); return true; @@ -15432,8 +16176,8 @@ class binary_reader } if (!value_in_range_of(number)) { - return sax->parse_error(chars_read, get_token_string(), out_of_range::create(408, - exception_message(input_format, "integer value overflow", "size"), nullptr)); + return report_error(chars_read, get_token_string(), out_of_range::create(408, + exception_message(input_format, "integer value overflow", "size"), nullptr)); } result = detail::conditional_static_cast(number); return true; @@ -15447,7 +16191,7 @@ class binary_reader } if (is_ndarray) // ndarray dimensional vector can only contain integers and cannot embed another array { - return sax->parse_error(chars_read, get_token_string(), parse_error::create(113, chars_read, exception_message(input_format, "ndarray dimensional vector is not allowed", "size"), nullptr)); + return report_error(chars_read, get_token_string(), parse_error::create(113, chars_read, exception_message(input_format, "ndarray dimensional vector is not allowed", "size"), nullptr)); } std::vector dim; if (JSON_HEDLEY_UNLIKELY(!get_ubjson_ndarray_size(dim))) @@ -15483,13 +16227,13 @@ class binary_reader // as modular arithmetic can produce any value, not just 0 or SIZE_MAX. if (JSON_HEDLEY_UNLIKELY(i > 0 && result > (std::numeric_limits::max)() / i)) { - return sax->parse_error(chars_read, get_token_string(), out_of_range::create(408, exception_message(input_format, "excessive ndarray size caused overflow", "size"), nullptr)); + return report_error(chars_read, get_token_string(), out_of_range::create(408, exception_message(input_format, "excessive ndarray size caused overflow", "size"), nullptr)); } result *= i; // Additional post-multiplication check to catch any edge cases the pre-check might miss if (result == 0 || result == npos) { - return sax->parse_error(chars_read, get_token_string(), out_of_range::create(408, exception_message(input_format, "excessive ndarray size caused overflow", "size"), nullptr)); + return report_error(chars_read, get_token_string(), out_of_range::create(408, exception_message(input_format, "excessive ndarray size caused overflow", "size"), nullptr)); } if (JSON_HEDLEY_UNLIKELY(!sax->number_unsigned(static_cast(i)))) { @@ -15517,7 +16261,7 @@ class binary_reader { message = "expected length type specification (U, i, u, I, m, l, M, L) after '#'; last byte: 0x" + last_token; } - return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format, message, "size"), nullptr)); + return report_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format, message, "size"), nullptr)); } /*! @@ -15551,8 +16295,8 @@ class binary_reader && JSON_HEDLEY_UNLIKELY(std::binary_search(bjd_optimized_type_markers.begin(), bjd_optimized_type_markers.end(), result.second))) { auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, - exception_message(input_format, concat("marker 0x", last_token, " is not a permitted optimized array type"), "type"), nullptr)); + return report_error(chars_read, last_token, parse_error::create(112, chars_read, + exception_message(input_format, concat("marker 0x", last_token, " is not a permitted optimized array type"), "type"), nullptr)); } if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "type"))) @@ -15568,8 +16312,8 @@ class binary_reader return false; } auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, - exception_message(input_format, concat("expected '#' after type information; last byte: 0x", last_token), "size"), nullptr)); + return report_error(chars_read, last_token, parse_error::create(112, chars_read, + exception_message(input_format, concat("expected '#' after type information; last byte: 0x", last_token), "size"), nullptr)); } const bool is_error = get_ubjson_size_value(result.first, is_ndarray); @@ -15588,8 +16332,8 @@ class binary_reader const bool is_error = get_ubjson_size_value(result.first, is_ndarray); if (input_format == input_format_t::bjdata && is_ndarray && !inside_ndarray) { - return sax->parse_error(chars_read, get_token_string(), parse_error::create(112, chars_read, - exception_message(input_format, "ndarray requires both type and size", "size"), nullptr)); + return report_error(chars_read, get_token_string(), parse_error::create(112, chars_read, + exception_message(input_format, "ndarray requires both type and size", "size"), nullptr)); } return is_error; } @@ -15765,8 +16509,8 @@ class binary_reader if (JSON_HEDLEY_UNLIKELY(current > 127)) { auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, - exception_message(input_format, concat("byte after 'C' must be in range 0x00..0x7F; last byte: 0x", last_token), "char"), nullptr)); + return report_error(chars_read, last_token, parse_error::create(113, chars_read, + exception_message(input_format, concat("byte after 'C' must be in range 0x00..0x7F; last byte: 0x", last_token), "char"), nullptr)); } string_t s(1, static_cast(current)); return sax->string(s); @@ -15788,7 +16532,7 @@ class binary_reader break; } auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format, "invalid byte: 0x" + last_token, "value"), nullptr)); + return report_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format, "invalid byte: 0x" + last_token, "value"), nullptr)); } /*! @@ -15816,8 +16560,8 @@ class binary_reader if (JSON_HEDLEY_UNLIKELY(it == bjd_types_map.end() || it->first != size_and_type.second)) { auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, - exception_message(input_format, "invalid byte: 0x" + last_token, "type"), nullptr)); + return report_error(chars_read, last_token, parse_error::create(112, chars_read, + exception_message(input_format, "invalid byte: 0x" + last_token, "type"), nullptr)); } string_t type = it->second; // sax->string() takes a reference @@ -15864,8 +16608,8 @@ class binary_reader if (JSON_HEDLEY_UNLIKELY((size_and_type.second == 'Z' || size_and_type.second == 'T' || size_and_type.second == 'F') && size_and_type.first > max_valueless_container_size)) { - return sax->parse_error(chars_read, get_token_string(), out_of_range::create(408, - exception_message(input_format, "excessive array size", "size"), nullptr)); + return report_error(chars_read, get_token_string(), out_of_range::create(408, + exception_message(input_format, "excessive array size", "size"), nullptr)); } if (JSON_HEDLEY_UNLIKELY(!enter_array(size_and_type.first, size_and_type.second))) @@ -15901,8 +16645,8 @@ class binary_reader if (input_format == input_format_t::bjdata && size_and_type.first != npos && (size_and_type.second & (1 << 8)) != 0) { auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, - exception_message(input_format, "BJData object does not support ND-array size in optimized format", "object"), nullptr)); + return report_error(chars_read, last_token, parse_error::create(112, chars_read, + exception_message(input_format, "BJData object does not support ND-array size in optimized format", "object"), nullptr)); } if (size_and_type.first != npos) @@ -15950,8 +16694,8 @@ class binary_reader if (JSON_HEDLEY_UNLIKELY(result_remainder != token_type::end_of_input)) { - return sax->parse_error(chars_read, number_string, parse_error::create(115, chars_read, - exception_message(input_format, concat("invalid number text: ", number_lexer.get_token_string()), "high-precision number"), nullptr)); + return report_error(chars_read, number_string, parse_error::create(115, chars_read, + exception_message(input_format, concat("invalid number text: ", number_lexer.get_token_string()), "high-precision number"), nullptr)); } switch (result_number) @@ -15965,7 +16709,7 @@ class binary_reader const auto parsed_float = number_lexer.get_number_float(); if (JSON_HEDLEY_UNLIKELY(!std::isfinite(parsed_float))) { - return sax->parse_error( + return report_error( chars_read, number_string, out_of_range::create(406, concat("number overflow parsing '", number_string, '\''), nullptr)); @@ -15990,8 +16734,8 @@ class binary_reader case token_type::end_of_input: case token_type::literal_or_value: default: - return sax->parse_error(chars_read, number_string, parse_error::create(115, chars_read, - exception_message(input_format, concat("invalid number text: ", number_lexer.get_token_string()), "high-precision number"), nullptr)); + return report_error(chars_read, number_string, parse_error::create(115, chars_read, + exception_message(input_format, concat("invalid number text: ", number_lexer.get_token_string()), "high-precision number"), nullptr)); } } @@ -16059,8 +16803,8 @@ class binary_reader bool bon8_error(const std::string& detail, const char* context) { auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, - exception_message(input_format_t::bon8, concat(detail, ": 0x", last_token), context), nullptr)); + return report_error(chars_read, last_token, parse_error::create(112, chars_read, + exception_message(input_format_t::bon8, concat(detail, ": 0x", last_token), context), nullptr)); } /*! @@ -16574,8 +17318,7 @@ class binary_reader { // in case of failure, advance position by 1 to report the failing location ++chars_read; - sax->parse_error(chars_read, "", parse_error::create(110, chars_read, exception_message(format, "unexpected end of input", context), nullptr)); - return false; + return report_error(chars_read, "", parse_error::create(110, chars_read, exception_message(format, "unexpected end of input", context), nullptr)); } return true; } @@ -16687,9 +17430,9 @@ class binary_reader // (which would defeat allow_exceptions=false / strict discarding). if (JSON_HEDLEY_UNLIKELY(!is_valid_utf8(result, old_size))) { - return sax->parse_error(chars_read, get_token_string(), - parse_error::create(113, chars_read, - exception_message(format, "invalid string: ill-formed UTF-8 byte", "string"), nullptr)); + return report_error(chars_read, get_token_string(), + parse_error::create(113, chars_read, + exception_message(format, "invalid string: ill-formed UTF-8 byte", "string"), nullptr)); } return true; @@ -16776,6 +17519,25 @@ class binary_reader return true; } + /*! + @brief report an error to the SAX parser + + The binary formats cannot recover from an error: a value's size is given + before its payload, and every byte value is a valid type marker, so after + an error there is no way to find where the next value begins. Reading + therefore stops, whatever the SAX parser's parse_error() returns. That the + SAX parser may ask for the containers read so far to be closed is handled + by @ref json_sax_salvager, not here (see #3989). + + @return false, so that the caller stops reading + */ + template + bool report_error(const std::size_t position, const std::string& last_token, const Exception& ex) const + { + static_cast(sax->parse_error(position, last_token, ex)); + return false; + } + /*! @param[in] format the current format (for diagnostics) @param[in] context further context information (for diagnostics) @@ -16786,8 +17548,8 @@ class binary_reader { if (JSON_HEDLEY_UNLIKELY(current == char_traits::eof())) { - return sax->parse_error(chars_read, "", - parse_error::create(110, chars_read, exception_message(format, "unexpected end of input", context), nullptr)); + return report_error(chars_read, "", + parse_error::create(110, chars_read, exception_message(format, "unexpected end of input", context), nullptr)); } return true; } @@ -17032,7 +17794,7 @@ class parser if (callback) { json_sax_dom_callback_parser sdp(result, callback, allow_exceptions, &m_lexer); - sax_parse_internal(&sdp); + sax_parse_internal(&sdp); if (strict) { @@ -17069,7 +17831,7 @@ class parser else { json_sax_dom_parser sdp(result, allow_exceptions, &m_lexer); - sax_parse_internal(&sdp); + sax_parse_internal(&sdp); if (strict) { @@ -17107,26 +17869,59 @@ class parser bool accept(const bool strict = true) { json_sax_acceptor sax_acceptor; - return sax_parse(&sax_acceptor, strict); + return sax_parse_impl(&sax_acceptor, strict); } + /*! + @brief public SAX interface + + If the SAX parser's parse_error() returns true, the parser recovers from + the error: it repairs the input and continues (see #3989). + + @param[in] sax the SAX parser + @param[in] strict whether to expect the last token to be EOF + @return whether the input was parsed without errors and no SAX event + returned false + */ template JSON_HEDLEY_NON_NULL(2) bool sax_parse(SAX* sax, const bool strict = true) + { + return sax_parse_impl(sax, strict); + } + + private: + /// what sax_parse_internal() does after an object key was expected + enum class next_step : std::uint8_t + { + /// stop parsing + stop, + /// parse a value that begins with last_token + parse_value, + /// evaluate the state of the innermost container, which reads + /// last_token again + evaluate_state + }; + + template + JSON_HEDLEY_NON_NULL(2) + bool sax_parse_impl(SAX* sax, const bool strict) { (void)detail::is_sax_static_asserts {}; - const bool result = sax_parse_internal(sax); + const bool result = sax_parse_internal(sax); if (result) { if (strict) { - // strict mode: next byte must be EOF - if (get_token() != token_type::end_of_input) + // strict mode: next byte must be EOF; after recovering from an + // error, the end of the input may already have been read + if (last_token != token_type::end_of_input && get_token() != token_type::end_of_input) { - return sax->parse_error(m_lexer.get_position(), - m_lexer.get_token_string(), - parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_of_input, "value"), nullptr)); + // the value is complete, so there is nothing to recover + static_cast(report_error(sax, parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_of_input, "value"), nullptr), + std::integral_constant {})); + return false; } } else @@ -17137,14 +17932,23 @@ class parser } } - return result; + return result && !error_reported; } - private: - template + /*! + @brief parse a JSON value and pass it to a SAX parser + + @tparam AllowRecovery whether to recover from an error if the SAX parser's + parse_error() returns true; false for the SAX parsers + of parse() and accept(), which never do, so that no + code for recovering is generated for them + */ + template JSON_HEDLEY_NON_NULL(2) bool sax_parse_internal(SAX* sax) { + const std::integral_constant allow_recovery{}; + // stack to remember the hierarchy of structured values we are parsing // true = array; false = object std::vector states; @@ -17175,12 +17979,18 @@ class parser break; } - // parse key + // remember we are now inside an object + states.push_back(false); + + // parse key (the steps of parse_key(), which are + // repeated here and below for speed) if (JSON_HEDLEY_UNLIKELY(last_token != token_type::value_string)) { - return sax->parse_error(m_lexer.get_position(), - m_lexer.get_token_string(), - parse_error::create(101, m_lexer.get_position(), exception_message(token_type::value_string, "object key"), nullptr)); + if (!continue_after(key_error(sax, allow_recovery, false), skip_to_state_evaluation)) + { + return false; + } + continue; } if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string()))) { @@ -17190,14 +18000,13 @@ class parser // parse separator (:) if (JSON_HEDLEY_UNLIKELY(get_token() != token_type::name_separator)) { - return sax->parse_error(m_lexer.get_position(), - m_lexer.get_token_string(), - parse_error::create(101, m_lexer.get_position(), exception_message(token_type::name_separator, "object separator"), nullptr)); + if (!continue_after(key_error(sax, allow_recovery, true), skip_to_state_evaluation)) + { + return false; + } + continue; } - // remember we are now inside an object - states.push_back(false); - // parse values get_token(); continue; @@ -17233,9 +18042,11 @@ class parser if (JSON_HEDLEY_UNLIKELY(!std::isfinite(res))) { - return sax->parse_error(m_lexer.get_position(), - m_lexer.get_token_string(), - out_of_range::create(406, concat("number overflow parsing '", m_lexer.get_token_string(), '\''), nullptr)); + if (!overflow_error(sax, res, allow_recovery)) + { + return false; + } + break; } if (JSON_HEDLEY_UNLIKELY(!sax->number_float(res, m_lexer.get_string()))) @@ -17303,23 +18114,63 @@ class parser case token_type::parse_error: { // using "uninitialized" to avoid an "expected" message - return sax->parse_error(m_lexer.get_position(), - m_lexer.get_token_string(), - parse_error::create(101, m_lexer.get_position(), exception_message(token_type::uninitialized, "value"), nullptr)); + if (!report_error(sax, parse_error::create(101, m_lexer.get_position(), exception_message(token_type::uninitialized, "value"), nullptr), allow_recovery)) + { + return false; + } + + // recover: keep what can be read of the token + recover_token(); + if (last_token != token_type::uninitialized) + { + // a string or a number + continue; + } + if (states.empty()) + { + // look for the value after the garbage + if (!skip_to_value()) + { + return false; + } + continue; + } + // nothing could be read + if (JSON_HEDLEY_UNLIKELY(!sax->null())) + { + return false; + } + break; } case token_type::end_of_input: { if (JSON_HEDLEY_UNLIKELY(m_lexer.get_position().chars_read_total == 1)) { - return sax->parse_error(m_lexer.get_position(), - m_lexer.get_token_string(), - parse_error::create(101, m_lexer.get_position(), - "attempting to parse an empty input; check that your input string or stream contains the expected JSON", nullptr)); + // there is nothing to recover + static_cast(report_error(sax, parse_error::create(101, m_lexer.get_position(), + "attempting to parse an empty input; check that your input string or stream contains the expected JSON", nullptr), allow_recovery)); + return false; } - return sax->parse_error(m_lexer.get_position(), - m_lexer.get_token_string(), - parse_error::create(101, m_lexer.get_position(), exception_message(token_type::literal_or_value, "value"), nullptr)); + if (!report_error(sax, parse_error::create(101, m_lexer.get_position(), exception_message(token_type::literal_or_value, "value"), nullptr), allow_recovery)) + { + return false; + } + + // recover: the input ends where a value is missing + if (states.empty()) + { + // there is no value + return false; + } + if (!recover_missing_value(sax, states)) + { + return false; + } + // the state evaluation reads the token again + m_lexer.unget_token(); + skip_to_state_evaluation = true; + continue; } case token_type::uninitialized: case token_type::end_array: @@ -17329,9 +18180,35 @@ class parser case token_type::literal_or_value: default: // the last token was unexpected { - return sax->parse_error(m_lexer.get_position(), - m_lexer.get_token_string(), - parse_error::create(101, m_lexer.get_position(), exception_message(token_type::literal_or_value, "value"), nullptr)); + if (!report_error(sax, parse_error::create(101, m_lexer.get_position(), exception_message(token_type::literal_or_value, "value"), nullptr), allow_recovery)) + { + return false; + } + + // recover + if (states.empty()) + { + // look for the value after the garbage + if (!skip_to_value()) + { + return false; + } + continue; + } + if (last_token == token_type::name_separator) + { + // a stray ':'; the value may follow + get_token(); + continue; + } + if (!recover_missing_value(sax, states)) + { + return false; + } + // the state evaluation reads the token again + m_lexer.unget_token(); + skip_to_state_evaluation = true; + continue; } } } @@ -17381,9 +18258,30 @@ class parser continue; } - return sax->parse_error(m_lexer.get_position(), - m_lexer.get_token_string(), - parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_array, "array"), nullptr)); + if (!report_error(sax, parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_array, "array"), nullptr), allow_recovery)) + { + return false; + } + + // recover + if (last_token == token_type::end_of_input) + { + // the input ends inside the array + return close_containers(sax, states); + } + if (last_token == token_type::end_object) + { + // a wrong closing bracket closes the innermost container + if (JSON_HEDLEY_UNLIKELY(!sax->end_array())) + { + return false; + } + states.pop_back(); + skip_to_state_evaluation = true; + } + // otherwise, a missing ',' (or a stray ':', which value + // parsing drops): the next value begins here + continue; } // states.back() is false -> object @@ -17400,11 +18298,12 @@ class parser // parse key if (JSON_HEDLEY_UNLIKELY(last_token != token_type::value_string)) { - return sax->parse_error(m_lexer.get_position(), - m_lexer.get_token_string(), - parse_error::create(101, m_lexer.get_position(), exception_message(token_type::value_string, "object key"), nullptr)); + if (!continue_after(key_error(sax, allow_recovery, false), skip_to_state_evaluation)) + { + return false; + } + continue; } - if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string()))) { return false; @@ -17413,9 +18312,11 @@ class parser // parse separator (:) if (JSON_HEDLEY_UNLIKELY(get_token() != token_type::name_separator)) { - return sax->parse_error(m_lexer.get_position(), - m_lexer.get_token_string(), - parse_error::create(101, m_lexer.get_position(), exception_message(token_type::name_separator, "object separator"), nullptr)); + if (!continue_after(key_error(sax, allow_recovery, true), skip_to_state_evaluation)) + { + return false; + } + continue; } // parse values @@ -17442,12 +18343,479 @@ class parser continue; } - return sax->parse_error(m_lexer.get_position(), - m_lexer.get_token_string(), - parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_object, "object"), nullptr)); + if (!report_error(sax, parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_object, "object"), nullptr), allow_recovery)) + { + return false; + } + + // recover + if (last_token == token_type::end_of_input) + { + // the input ends inside the object + return close_containers(sax, states); + } + if (last_token == token_type::end_array) + { + // a wrong closing bracket closes the innermost container + if (JSON_HEDLEY_UNLIKELY(!sax->end_object())) + { + return false; + } + states.pop_back(); + skip_to_state_evaluation = true; + continue; + } + if (!continue_after(recover_member(sax, allow_recovery), skip_to_state_evaluation)) + { + return false; + } } } + /*! + @brief continue sax_parse_internal() after a recovery + @return whether to continue parsing + */ + bool continue_after(const next_step step, bool& skip_to_state_evaluation) + { + if (step == next_step::evaluate_state) + { + // the state evaluation reads the token again + m_lexer.unget_token(); + skip_to_state_evaluation = true; + } + return step != next_step::stop; + } + + /// the parser for parse() and accept() never recovers: stop parsing + static std::false_type continue_after(std::false_type /*step*/, bool& /*skip_to_state_evaluation*/) noexcept + { + return {}; + } + + /*! + @brief parse an object key and the name separator (:) after it + + last_token is the token where the key is expected. sax_parse_internal() + repeats these steps rather than calling this function, which is used + when recovering from an error. + + @return next_step::parse_value if the value follows, with last_token its + first token; next_step::evaluate_state if the object's state is + to be evaluated after recovering from an error; next_step::stop + to stop parsing + */ + template + next_step parse_key(SAX* sax) + { + const std::true_type allow_recovery{}; + + if (JSON_HEDLEY_UNLIKELY(last_token != token_type::value_string)) + { + return key_error(sax, allow_recovery, false); + } + + if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string()))) + { + return next_step::stop; + } + + // parse separator (:) + if (JSON_HEDLEY_UNLIKELY(get_token() != token_type::name_separator)) + { + return key_error(sax, allow_recovery, true); + } + + // the value begins with the next token + get_token(); + return next_step::parse_value; + } + + /*! + @brief report a number that is too large for number_float_t, and recover + from the error by passing the value on; the SAX parser gets the + number's text as well + + This is a separate function, as reading other numbers is measurably + slower if the error is handled where they are read. + + @param[in] sax the SAX parser + @param[in] value the value that is not finite + @return whether to continue parsing + */ + template + bool overflow_error(SAX* sax, const number_float_t value, AllowRecovery allow_recovery) + { + if (!report_error(sax, out_of_range::create(406, concat("number overflow parsing '", m_lexer.get_token_string(), '\''), nullptr), allow_recovery)) + { + return false; + } + return sax->number_float(value, m_lexer.get_string()); + } + + /*! + @brief report a missing key, or a missing name separator (:) after the + key; the parser for parse() and accept() never recovers + + @param[in] key_read whether the key was read, so that the name separator + is missing + @return std::false_type, see report_error() + */ + template + std::false_type key_error(SAX* sax, std::false_type allow_recovery, const bool key_read) + { + return report_error(sax, parse_error::create(101, m_lexer.get_position(), key_read + ? exception_message(token_type::name_separator, "object separator") + : exception_message(token_type::value_string, "object key"), nullptr), allow_recovery); + } + + /*! + @brief report a missing key, or a missing name separator (:) after the + key, and recover from it + + @param[in] key_read whether the key was read, so that the name separator + is missing + */ + template + next_step key_error(SAX* sax, std::true_type allow_recovery, const bool key_read) + { + if (!key_read) + { + if (!report_error(sax, parse_error::create(101, m_lexer.get_position(), exception_message(token_type::value_string, "object key"), nullptr), allow_recovery)) + { + return next_step::stop; + } + return recover_key(sax); + } + + if (!report_error(sax, parse_error::create(101, m_lexer.get_position(), exception_message(token_type::name_separator, "object separator"), nullptr), allow_recovery)) + { + return next_step::stop; + } + return recover_name_separator(sax); + } + + ///////////////////// + // error recovery + ///////////////////// + + /* + The functions below repair an error after the SAX parser's parse_error() + returned true (see #3989). Each mistake is repaired by the smallest local + edit: a missing ',' or ':' is inserted, a stray token is removed, what can + be read of an invalid string or number is kept (see + lexer::recover_token()), a missing value becomes null, a wrong closing + bracket closes the innermost container, and the end of the input closes + all of them. The events stay balanced, and every key() is followed by + exactly one value. + + A repair hands a token to the state evaluation, by returning it to the + lexer (lexer::unget_token()) so that the state evaluation reads it again, + only if it is ',', ']', '}', or the end of the input. The state evaluation + hands a token to value or key parsing only if it is none of them, so a + token is never handed back and forth. Every other step reads a token or + closes a container, so parsing always ends. + */ + + /*! + @brief report an error to the SAX parser; the parser for parse() and + accept() never recovers + + @return std::false_type rather than false: its value is known where the + function is called even if the call is not inlined, so the code + for recovering is not generated + */ + template + std::false_type report_error(SAX* sax, const Exception& ex, std::false_type /*allow_recovery*/) + { + error_reported = true; + static_cast(sax->parse_error(m_lexer.get_position(), m_lexer.get_token_string(), ex)); + return {}; + } + + /*! + @brief report an error to the SAX parser + @return whether to recover from the error + */ + template + bool report_error(SAX* sax, const Exception& ex, std::true_type /*allow_recovery*/) + { + const std::size_t position = m_lexer.get_position().chars_read_total; + if (error_reported && position == last_error_position && last_token == last_error_token) + { + // a repair handed on the token of the error it repaired; the + // token was reported already, and the SAX parser asked to recover + return true; + } + + error_reported = true; + last_error_position = position; + last_error_token = last_token; + + if (!sax->parse_error(m_lexer.get_position(), m_lexer.get_token_string(), ex)) + { + return false; + } + + // the token string of the next error begins here + m_lexer.restart_token_string(); + return true; + } + + /*! + @brief keep what can be read of the token that the lexer rejected + + The error was reported for the rejected token, so it is not reported again + for the token it is repaired to (see lexer::recover_token()). + */ + token_type recover_token() + { + last_token = m_lexer.recover_token(); + last_error_position = m_lexer.get_position().chars_read_total; + last_error_token = last_token; + return last_token; + } + + /// pass the end events of all open containers + template + bool close_containers(SAX* sax, std::vector& states) + { + while (!states.empty()) + { + const bool is_array = states.back(); + states.pop_back(); + if (JSON_HEDLEY_UNLIKELY(is_array ? !sax->end_array() : !sax->end_object())) + { + return false; + } + } + return true; + } + + /*! + @brief read tokens until one begins a value, skipping everything before + the top-level value + @return whether a value begins with last_token + */ + bool skip_to_value() + { + while (true) + { + switch (get_token()) + { + case token_type::begin_array: + case token_type::begin_object: + case token_type::literal_false: + case token_type::literal_null: + case token_type::literal_true: + case token_type::value_float: + case token_type::value_integer: + case token_type::value_string: + case token_type::value_unsigned: + return true; + + case token_type::end_of_input: + return false; + + case token_type::parse_error: + recover_token(); + if (last_token != token_type::uninitialized) + { + return true; + } + break; + + case token_type::uninitialized: + case token_type::end_array: + case token_type::end_object: + case token_type::name_separator: + case token_type::value_separator: + case token_type::literal_or_value: + default: + break; + } + } + } + + /*! + @brief skip the rest of an object member that cannot be read + + Reads tokens, beginning with last_token, until a ',', '}', or ']' that is + not inside a container that begins in the skipped tokens, or the end of + the input. + */ + void skip_member() + { + std::size_t depth = 0; + while (true) + { + switch (last_token) + { + case token_type::begin_array: + case token_type::begin_object: + ++depth; + break; + + case token_type::end_array: + case token_type::end_object: + if (depth == 0) + { + return; + } + --depth; + break; + + case token_type::value_separator: + if (depth == 0) + { + return; + } + break; + + case token_type::end_of_input: + return; + + case token_type::parse_error: + recover_token(); + break; + + case token_type::uninitialized: + case token_type::literal_true: + case token_type::literal_false: + case token_type::literal_null: + case token_type::value_string: + case token_type::value_unsigned: + case token_type::value_integer: + case token_type::value_float: + case token_type::name_separator: + case token_type::literal_or_value: + default: + break; + } + get_token(); + } + } + + /*! + @brief pass a value where it is missing + + last_token is ',', ']', '}', or the end of the input, where a value was + expected. In an object, the key gets null; in an array, a ',' where a + value is missing stands for null (as in JavaScript), while an array that + ends there just ends. + */ + template + bool recover_missing_value(SAX* sax, const std::vector& states) + { + JSON_ASSERT(!states.empty()); + if (!states.back() || last_token == token_type::value_separator) + { + return sax->null(); + } + return true; + } + + /// recover from a missing key; last_token is where it was expected + template + next_step recover_key(SAX* sax) + { + switch (last_token) + { + case token_type::value_separator: + case token_type::end_object: + case token_type::end_array: + case token_type::end_of_input: + // no member: the object's state handles the token + return next_step::evaluate_state; + + case token_type::parse_error: + recover_token(); + if (last_token == token_type::value_string) + { + // a key that could be repaired + return parse_key(sax); + } + skip_member(); + return next_step::evaluate_state; + + case token_type::uninitialized: + case token_type::literal_true: + case token_type::literal_false: + case token_type::literal_null: + case token_type::value_string: + case token_type::value_unsigned: + case token_type::value_integer: + case token_type::value_float: + case token_type::begin_array: + case token_type::begin_object: + case token_type::name_separator: + case token_type::literal_or_value: + default: + // a member without a key + skip_member(); + return next_step::evaluate_state; + } + } + + /// recover from a missing name separator (:) after the key; last_token + /// is where it was expected + template + next_step recover_name_separator(SAX* sax) + { + switch (last_token) + { + case token_type::value_separator: + case token_type::end_object: + case token_type::end_array: + case token_type::end_of_input: + // the value is missing as well + return sax->null() ? next_step::evaluate_state : next_step::stop; + + case token_type::uninitialized: + case token_type::literal_true: + case token_type::literal_false: + case token_type::literal_null: + case token_type::value_string: + case token_type::value_unsigned: + case token_type::value_integer: + case token_type::value_float: + case token_type::begin_array: + case token_type::begin_object: + case token_type::name_separator: + case token_type::parse_error: + case token_type::literal_or_value: + default: + // a missing ':'; the value begins here + return next_step::parse_value; + } + } + + /// recover from a token after an object member that is neither ',' nor + /// '}' (nor ']' or the end of the input, which the caller handles) + template + next_step recover_member(SAX* sax, std::true_type /*allow_recovery*/) + { + if (last_token == token_type::parse_error) + { + recover_token(); + } + if (last_token == token_type::value_string) + { + // a missing ','; the next key begins here + return parse_key(sax); + } + skip_member(); + return next_step::evaluate_state; + } + + /// the parser for parse() and accept() never recovers (and does not come + /// here, as report_error() returned false) + template + std::false_type recover_member(SAX* /*sax*/, std::false_type /*allow_recovery*/) const noexcept + { + return {}; + } + /// get next token from lexer token_type get_token() { @@ -17494,6 +18862,12 @@ class parser const bool allow_exceptions = true; /// whether trailing commas in objects and arrays should be ignored (true) or signaled as errors (false) const bool ignore_trailing_commas = false; + /// whether an error was reported to the SAX parser + bool error_reported = false; + /// the position of the last reported error + std::size_t last_error_position = 0; + /// the token of the last reported error + token_type last_error_token = token_type::uninitialized; }; } // namespace detail @@ -30862,6 +32236,26 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec return parser(i.get(), nullptr, false, ignore_comments, ignore_trailing_commas, true).accept(true); } + private: + /// read a binary format and pass it to a SAX parser; if the SAX parser + /// asks to recover from an error, the value read so far is completed + /// (see detail::json_sax_salvager and #3989) + template + static bool sax_parse_binary(InputAdapterType ia, SAX* sax, + const input_format_t format, const bool strict) + { + (void)detail::is_sax_static_asserts {}; + using salvager_t = detail::json_sax_salvager; + salvager_t salvager(sax); + const bool result = detail::binary_reader(std::move(ia), format).sax_parse(format, &salvager, strict); + if (!result) + { + salvager.close_open_containers(); + } + return result; + } + + public: /// @brief generate SAX events /// @sa https://json.nlohmann.me/api/basic_json/sax_parse/ template @@ -30875,7 +32269,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec auto ia = detail::input_adapter(std::forward(i)); return format == input_format_t::json ? parser(std::move(ia), nullptr, true, ignore_comments, ignore_trailing_commas).sax_parse(sax, strict) - : detail::binary_reader(std::move(ia), format).sax_parse(format, sax, strict); + : sax_parse_binary(std::move(ia), sax, format, strict); } /// @brief generate SAX events (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -30892,7 +32286,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec auto ia = detail::input_adapter(std::move(first), std::move(last)); return format == input_format_t::json ? parser(std::move(ia), nullptr, true, ignore_comments, ignore_trailing_commas).sax_parse(sax, strict) - : detail::binary_reader(std::move(ia), format).sax_parse(format, sax, strict); + : sax_parse_binary(std::move(ia), sax, format, strict); } /// @brief generate SAX events @@ -30914,7 +32308,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) ? parser(std::move(ia), nullptr, true, ignore_comments, ignore_trailing_commas).sax_parse(sax, strict) // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) - : detail::binary_reader(std::move(ia), format).sax_parse(format, sax, strict); + : sax_parse_binary(std::move(ia), sax, format, strict); } #ifndef JSON_NO_IO /// @brief deserialize from stream diff --git a/src/modules/json.cppm b/src/modules/json.cppm index 15207535f..e364dd82d 100644 --- a/src/modules/json.cppm +++ b/src/modules/json.cppm @@ -46,6 +46,7 @@ inline namespace json_literals namespace detail { using NLOHMANN_JSON_NAMESPACE::detail::json_sax_dom_callback_parser; + using NLOHMANN_JSON_NAMESPACE::detail::json_sax_dom_parser; using NLOHMANN_JSON_NAMESPACE::detail::unknown_size; } // namespace detail diff --git a/tests/src/fuzzer-parse_json.cpp b/tests/src/fuzzer-parse_json.cpp index 217d9e0fd..4644fa969 100644 --- a/tests/src/fuzzer-parse_json.cpp +++ b/tests/src/fuzzer-parse_json.cpp @@ -16,6 +16,10 @@ array data, it performs the following steps: - s2 = serialize(j2) - assert(s1 == s2) +Furthermore, it parses data with a SAX parser that recovers from every error +and checks that the events are balanced, that parsing ends, and that valid +input is parsed without errors (see #3989). + The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer drivers. */ @@ -23,6 +27,8 @@ drivers. #include #include #include +#include +#include #include // the round-trip checks below are assertions; NDEBUG would compile them away @@ -32,9 +38,143 @@ drivers. using json = nlohmann::json; +namespace +{ +// a SAX parser that recovers from every error and checks that the events are +// balanced and that every key is followed by exactly one value +class recovering_checker : public nlohmann::json_sax +{ + public: + bool null() override + { + return value(); + } + + bool boolean(bool /*val*/) override + { + return value(); + } + + bool number_integer(number_integer_t /*val*/) override + { + return value(); + } + + bool number_unsigned(number_unsigned_t /*val*/) override + { + return value(); + } + + bool number_float(number_float_t /*val*/, const string_t& /*s*/) override + { + return value(); + } + + bool string(string_t& /*val*/) override + { + return value(); + } + + bool binary(binary_t& /*val*/) override + { + return value(); + } + + bool start_object(std::size_t /*elements*/) override + { + value(); + stack.push_back('o'); + return true; + } + + bool key(string_t& /*val*/) override + { + ++events; + assert(!stack.empty() && stack.back() == 'o'); + stack.back() = 'v'; + return true; + } + + bool end_object() override + { + ++events; + assert(!stack.empty() && stack.back() == 'o'); + stack.pop_back(); + return true; + } + + bool start_array(std::size_t /*elements*/) override + { + value(); + stack.push_back('a'); + return true; + } + + bool end_array() override + { + ++events; + assert(!stack.empty() && stack.back() == 'a'); + stack.pop_back(); + return true; + } + + bool parse_error(std::size_t /*position*/, const std::string& /*last_token*/, const nlohmann::detail::exception& /*ex*/) override + { + ++errors; + return true; + } + + bool complete() const + { + return stack.empty(); + } + + std::size_t events = 0; + std::size_t errors = 0; + + private: + bool value() + { + ++events; + if (!stack.empty()) + { + // an array element, or the value of a key + assert(stack.back() != 'o'); + if (stack.back() == 'v') + { + stack.back() = 'o'; + } + } + return true; + } + + // 'a' for an array, 'o' for an object that expects a key, 'v' for an + // object that expects the value of a key + std::vector stack; +}; +} // namespace + // see http://llvm.org/docs/LibFuzzer.html extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { + // step 0: recover from all errors, reading from memory and from a stream + { + recovering_checker checker; + const bool ok = json::sax_parse(data, data + size, &checker); + assert(checker.complete()); + assert(checker.errors <= size + 1); + assert(checker.events <= (4 * size) + 4); + assert(ok == json::accept(data, data + size)); + assert(ok == (checker.errors == 0)); + + std::istringstream stream(std::string(reinterpret_cast(data), size)); + recovering_checker stream_checker; + assert(json::sax_parse(stream, &stream_checker) == ok); + assert(stream_checker.complete()); + assert(stream_checker.events == checker.events); + assert(stream_checker.errors == checker.errors); + } + try { // step 1: parse input diff --git a/tests/src/unit-class_parser.cpp b/tests/src/unit-class_parser.cpp index 3d825162e..16840d93d 100644 --- a/tests/src/unit-class_parser.cpp +++ b/tests/src/unit-class_parser.cpp @@ -2817,3 +2817,586 @@ TEST_CASE("diagnostic positions: value lifetime, input adapters, and SAX") } } #endif + +namespace +{ +/// builds a value like json::parse(), but asks the parser to recover from +/// errors (see #3989), and checks that the events it receives are balanced +class RecoveringDomParser : public nlohmann::detail::json_sax_dom_parser +{ + using base = nlohmann::detail::json_sax_dom_parser; + + public: + explicit RecoveringDomParser(json& j, std::size_t max_errors_ = static_cast(-1)) + : base(j, false) + , max_errors(max_errors_) + {} + + bool null() + { + value(); + return base::null(); + } + + bool boolean(bool val) + { + value(); + return base::boolean(val); + } + + bool number_integer(json::number_integer_t val) + { + value(); + return base::number_integer(val); + } + + bool number_unsigned(json::number_unsigned_t val) + { + value(); + return base::number_unsigned(val); + } + + bool number_float(json::number_float_t val, const std::string& s) + { + value(); + return base::number_float(val, s); + } + + bool string(std::string& val) + { + value(); + return base::string(val); + } + + bool start_object(std::size_t elements) + { + value(); + stack.push_back('o'); + return base::start_object(elements); + } + + bool key(std::string& val) + { + ++events; + if (stack.empty() || stack.back() != 'o') + { + well_formed = false; + return false; + } + stack.back() = 'v'; + return base::key(val); + } + + bool end_object() + { + ++events; + if (stack.empty() || stack.back() != 'o') + { + well_formed = false; + return false; + } + stack.pop_back(); + return base::end_object(); + } + + bool start_array(std::size_t elements) + { + value(); + stack.push_back('a'); + return base::start_array(elements); + } + + bool end_array() + { + ++events; + if (stack.empty() || stack.back() != 'a') + { + well_formed = false; + return false; + } + stack.pop_back(); + return base::end_array(); + } + + bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, const json::exception& ex) + { + errors.emplace_back(ex.what()); + return errors.size() < max_errors; + } + + /// whether the events were balanced and every key was followed by a value + bool balanced() const + { + return well_formed && stack.empty(); + } + + std::vector errors {}; // NOLINT(readability-redundant-member-init) + std::size_t events = 0; + /// the open containers: 'a' for an array, 'o' for an object that expects + /// a key, 'v' for an object that expects the value of a key + std::vector stack {}; // NOLINT(readability-redundant-member-init) + bool well_formed = true; + std::size_t max_errors; + + private: + /// a value is passed: it is an array element, or the value of a key + void value() + { + ++events; + if (!stack.empty()) + { + if (stack.back() == 'v') + { + stack.back() = 'o'; + } + else if (stack.back() == 'o') + { + // a value without a key + well_formed = false; + } + } + } + +}; + +struct RecoveryResult +{ + json value; + std::vector errors; + std::size_t events; + bool ok; + bool balanced; +}; + +template +RecoveryResult parse_recovering(InputType&& input, const bool strict = true, + const bool ignore_comments = false, const bool ignore_trailing_commas = false) +{ + json j; + RecoveringDomParser sax(j); + const bool ok = json::sax_parse(std::forward(input), &sax, json::input_format_t::json, + strict, ignore_comments, ignore_trailing_commas); + return {j, sax.errors, sax.events, ok, sax.balanced()}; +} + +/// logs the events as strings and recovers from errors +class RecoveringEventLogger : public SaxEventLogger +{ + public: + bool parse_error(std::size_t position, const std::string& /*unused*/, const json::exception& /*unused*/) + { + events.push_back("parse_error(" + std::to_string(position) + ")"); + return true; + } +}; + +/// stops after a number of events, but recovers from errors +class RecoveringCountdown : public SaxCountdown +{ + public: + using SaxCountdown::SaxCountdown; + + bool parse_error(std::size_t /*position*/, const std::string& /*last_token*/, const json::exception& /*ex*/) override + { + return true; + } +}; + +/// a repaired input: the value it is repaired to, and the number of errors +struct Repair +{ + const char* input; + const char* expected; + std::size_t errors; +}; +} // namespace + +TEST_CASE("parser error recovery (#3989)") +{ + SECTION("repairs") + { + const std::vector repairs = + { + // a missing separator is inserted + {"[1 2]", "[1,2]", 1}, + {R"({"a":1 "b":2})", R"({"a":1,"b":2})", 1}, + {R"({"a" 1})", R"({"a":1})", 1}, + {"[1 tru 2]", "[1,null,2]", 2}, + {R"({"a" "b": 1})", R"({"a":"b"})", 2}, + + // a missing value is null in an object; in an array, a ',' stands + // for null, while an array that ends there just ends + {R"({"a":})", R"({"a":null})", 1}, + {R"({"a"})", R"({"a":null})", 1}, + {R"({"a","b":1})", R"({"a":null,"b":1})", 1}, + {"[1,,2]", "[1,null,2]", 1}, + {"[,1]", "[null,1]", 1}, + {"[1,]", "[1]", 1}, + {"[1,2,3,]", "[1,2,3]", 1}, + {R"({"a":1,})", R"({"a":1})", 1}, + + // a broken string keeps what can be read + {R"(["a\qb"])", R"(["aqb"])", 1}, + {R"({"na\me":1})", R"({"name":1})", 1}, + {"[\"\xFF\"]", R"(["\uFFFD"])", 1}, + {"[\"a\xC3(\"]", R"(["a\uFFFD("])", 1}, + {"[\"\xE2\x82\"]", R"(["\uFFFD"])", 1}, + {"[\"\xC3\\\\\", 1]", R"(["\uFFFD\\",1])", 1}, + {R"(["\u12"])", R"(["\uFFFD"])", 1}, + {R"(["\u12G4"])", R"(["\uFFFDG4"])", 1}, + {R"(["\uDC00x"])", R"(["\uFFFDx"])", 1}, + {R"(["\uD800x"])", R"(["\uFFFDx"])", 1}, + {R"(["\uD800\u0041"])", R"(["\uFFFDA"])", 1}, + {R"(["\uD800\uD800\uDC00"])", R"(["\uFFFD\uD800\uDC00"])", 1}, + {R"(["\uD800\uD800\uD800x"])", R"(["\uFFFD\uFFFD\uFFFDx"])", 1}, + { + R"(["\uD800\"x", 1])", R"(["\uFFFD\"x",1])", 1 + }, + {R"(["\uD800\q"])", R"(["\uFFFDq"])", 1}, + {"[\"a\tb\"]", R"(["a\tb"])", 1}, + {R"(["a\qb\u0041\x"])", R"(["aqbAx"])", 1}, + + // a broken number keeps its longest valid prefix + {"[1.]", "[1]", 1}, + {"[-2.]", "[-2]", 1}, + {"[1.5e]", "[1.5]", 1}, + {"[1e+]", "[1]", 1}, + {"[1.x2, 3]", "[1,3]", 1}, + + // what cannot be read at all is null + {"[1,NaN,3]", "[1,null,3]", 1}, + {"[tru]", "[null]", 1}, + {"[-]", "[null]", 1}, + {R"({"a":Infinity})", R"({"a":null})", 1}, + + // a stray token is dropped + {"[:1]", "[1]", 1}, + {R"(["a":1])", R"(["a",1])", 1}, + {R"({"a"::1})", R"({"a":1})", 1}, + + // a member that cannot be read is skipped + {R"({1:2,"b":3})", R"({"b":3})", 1}, + {R"({"a":1 2})", R"({"a":1})", 1}, + {R"({,"a":1})", R"({"a":1})", 1}, + {R"({"a":1,,"b":2})", R"({"a":1,"b":2})", 1}, + {"{a:1}", "{}", 1}, + {R"({"a":1 [1,{"b":2}], "c":3})", R"({"a":1,"c":3})", 1}, + {R"([{1}, "a"])", R"([{},"a"])", 1}, + + // a wrong closing bracket closes the innermost container + {R"({"a":[1,2}, "b":3})", R"({"a":[1,2],"b":3})", 1}, + {R"([{"a":1], 2])", R"([{"a":1},2])", 1}, + {"{]", "{}", 1}, + {"[}", "[]", 1}, + + // the end of the input closes all containers + {R"({"a":[1,2)", R"({"a":[1,2]})", 1}, + {"[", "[]", 1}, + {"{", "{}", 1}, + {R"({"a")", R"({"a":null})", 1}, + {R"({"a":)", R"({"a":null})", 1}, + {"[1,", "[1]", 1}, + {"[[[1", "[[[1]]]", 1}, + { + R"(["abc)", R"(["abc"])", 2 + }, + {"[1,tr", "[1,null]", 2}, + {"\"abc", "\"abc\"", 1}, + {"[\"ab\ncd\"]", R"(["ab",null,"]"])", 4}, + + // what comes before the top-level value is skipped + {")]}'\n{\"a\":1}", R"({"a":1})", 1}, + {R"(data: {"a":1})", R"({"a":1})", 1}, + {"\xEF\xBB[1]", "[1]", 1}, + + // what comes after it is an error that ends parsing + {R"({"a":1}})", R"({"a":1})", 1}, + {"[1}]", "[1]", 2}, + {"[1] [2]", "[1]", 1}, + }; + + for (const auto& repair : repairs) + { + CAPTURE(repair.input); + const auto result = parse_recovering(std::string(repair.input)); + CHECK(!result.ok); + CHECK(result.balanced); + CHECK(result.value == json::parse(repair.expected)); + CHECK(result.errors.size() == repair.errors); + } + } + + SECTION("number overflow") + { + const auto result = parse_recovering(std::string("[1e999,-1e999]")); + CHECK(!result.ok); + CHECK(result.balanced); + CHECK(result.errors.size() == 2); + CHECK(result.errors[0] == "[json.exception.out_of_range.406] number overflow parsing '1e999'"); + REQUIRE(result.value.size() == 2); + CHECK(result.value[0].is_number_float()); + CHECK(result.value[0].get() == std::numeric_limits::infinity()); + CHECK(result.value[1].get() == -std::numeric_limits::infinity()); + + // the SAX parser gets the number's text + RecoveringEventLogger logger; + CHECK(!json::sax_parse("1e999", &logger)); + CHECK(logger.events == std::vector({"parse_error(5)", "number_float(1e999)"})); + } + + SECTION("nothing to recover") + { + for (const std::string s : + { + "", " ", "]", "tru", "NaN", ",:", "/* comment" + }) + { + CAPTURE(s); + const auto result = parse_recovering(s, true, true); + CHECK(!result.ok); + CHECK(result.balanced); + CHECK(result.events == 0); + CHECK(result.value == nullptr); + CHECK(result.errors.size() == 1); + } + } + + SECTION("error messages") + { + // the first error is reported as without recovery + for (const std::string s : + { + "[1 2]", R"({"a":1 "b":2})", R"({"a" 1})", R"({"a":})", "[1,]", "[1.]", + R"(["a\qb"])", "[1e999]", "{1:2}", R"({"a":[1,2}})", "[1,", "[1] [2]", "{a:1}" + }) + { + CAPTURE(s); + const auto result = parse_recovering(s); + REQUIRE(!result.errors.empty()); + json _; + CHECK_THROWS_WITH_STD_STR(_ = json::parse(s), result.errors.front()); + } + + // the token of an error begins where the previous error was + const auto result = parse_recovering(std::string("[tru, fals, nul]")); + CHECK(result.errors == std::vector( + { + "[json.exception.parse_error.101] parse error at line 1, column 5: syntax error while parsing value - invalid literal; last read: '[tru,'", + "[json.exception.parse_error.101] parse error at line 1, column 11: syntax error while parsing value - invalid literal; last read: ', fals,'", + "[json.exception.parse_error.101] parse error at line 1, column 16: syntax error while parsing value - invalid literal; last read: ', nul]'" + })); + CHECK(result.value == json::parse("[null,null,null]")); + } + + SECTION("events") + { + // see #4522 + RecoveringEventLogger logger; + CHECK(!json::sax_parse(R"([{1}, "a"])", &logger)); + CHECK(logger.events == std::vector( + { + "start_array()", "start_object()", "parse_error(3)", "end_object()", "string(a)", "end_array()" + })); + } + + SECTION("options") + { + SECTION("strict") + { + const auto result = parse_recovering(std::string("[1 2] [3]"), false); + CHECK(!result.ok); + CHECK(result.value == json::parse("[1,2]")); + CHECK(result.errors.size() == 1); + } + + SECTION("ignore_trailing_commas") + { + for (const std::string s : + { + "[1,]", R"({"a":1,})", "[[1,],]" + }) + { + CAPTURE(s); + const auto result = parse_recovering(s, true, false, true); + CHECK(result.ok); + CHECK(result.errors.empty()); + } + + auto result = parse_recovering(std::string("[1,,]"), true, false, true); + CHECK(result.value == json::parse("[1,null]")); + CHECK(result.errors.size() == 1); + + result = parse_recovering(std::string(R"({"a":1,,})"), true, false, true); + CHECK(result.value == json::parse(R"({"a":1})")); + CHECK(result.errors.size() == 1); + } + + SECTION("ignore_comments") + { + auto result = parse_recovering(std::string("[1 /* one */ 2]"), true, true); + CHECK(result.value == json::parse("[1,2]")); + CHECK(result.errors.size() == 1); + + // a comment that is not closed runs to the end of the input, which + // is not reported again + result = parse_recovering(std::string("[1, 2 /* unterminated"), true, true); + CHECK(result.balanced); + CHECK(result.value == json::parse("[1,2]")); + CHECK(result.errors.size() == 1); + + // a '/' that does not begin a comment is garbage + result = parse_recovering(std::string("[1, /x, 2]"), true, true); + CHECK(result.balanced); + CHECK(result.value == json::parse("[1,null,2]")); + CHECK(result.errors.size() == 1); + } + } + + SECTION("null bytes") + { + // a null byte ends the input, unless JSON_STRICT_NUL_HANDLING is set + const auto result = parse_recovering(std::string("[1,\0x", 5)); + CHECK(result.balanced); + CHECK(!result.ok); +#ifdef JSON_TEST_STRICT_NUL_HANDLING_ENABLED + CHECK(result.value == json::parse("[1,null]")); +#else + CHECK(result.value == json::parse("[1]")); + CHECK(result.errors.size() == 1); +#endif + + const auto in_string = parse_recovering(std::string("[\"a\0b\"]", 7)); + CHECK(in_string.balanced); +#ifdef JSON_TEST_STRICT_NUL_HANDLING_ENABLED + CHECK(in_string.value == json::parse(R"(["a\u0000b"])")); +#else + CHECK(in_string.value == json::parse(R"(["a"])")); +#endif + } + + SECTION("the SAX parser stops recovering") + { + json j; + RecoveringDomParser sax(j, 2); + CHECK(!json::sax_parse("[1 2 3 4 5]", &sax)); + CHECK(sax.errors.size() == 2); + + // an error at a delimiter that an invalid token consumed is reported + // to the SAX parser, too + json j2; + RecoveringDomParser sax2(j2, 2); + CHECK(!json::sax_parse("[tru}, 1]", &sax2)); + CHECK(sax2.errors.size() == 2); + } + + SECTION("an event stops parsing during a repair") + { + // start_object() and key() are passed, then null() for the missing + // value returns false + RecoveringCountdown countdown(2); + CHECK(!json::sax_parse(R"({"a":})", &countdown)); + + // the end of the input: end_array() for the second array returns false + RecoveringCountdown countdown2(4); + CHECK(!json::sax_parse("[[1", &countdown2)); + } + + SECTION("input adapters") + { + // the lexer reads contiguous and streaming input differently, and it + // puts back a character that ended an invalid token + for (const std::string s : + { + "[1 2]", "[tru}, 1]", R"({"a" "b\q", "c":[1.x, 2}})", "[\"\xFF\xC3(\", -, 1e+]", "{a:1,\"b\":2", ")]}' [1]" + }) + { + CAPTURE(s); + const auto reference = parse_recovering(s); + CHECK(reference.balanced); + + const auto from_c_string = parse_recovering(s.c_str()); + CHECK(from_c_string.value == reference.value); + CHECK(from_c_string.errors == reference.errors); + + const std::list l(s.begin(), s.end()); + json j; + RecoveringDomParser sax(j); + CHECK(!json::sax_parse(l.begin(), l.end(), &sax)); + CHECK(j == reference.value); + CHECK(sax.errors == reference.errors); + + std::istringstream ss(s); + const auto from_stream = parse_recovering(ss); + CHECK(from_stream.value == reference.value); + CHECK(from_stream.errors == reference.errors); + } + } + + SECTION("long runs of errors") + { + // no error may copy all the input read before it + const auto closing = parse_recovering("[" + std::string(100000, '}')); + CHECK(closing.balanced); + CHECK(closing.value == json::array()); + + const auto garbage = parse_recovering("[" + std::string(100000, 'x') + "]"); + CHECK(garbage.balanced); + CHECK(garbage.errors.size() == 1); + + const auto commas = parse_recovering("{" + std::string(100000, ',') + "}"); + CHECK(commas.balanced); + CHECK(commas.value == json::object()); + } + + SECTION("mutations of valid input") + { + // whatever the input, the events are balanced, every error is reported + // at most once, and valid input is parsed as usual + const std::vector documents = + { + R"({"name": "value", "list": [1, -2.5, true, null, {"x": [[]]}], "e": "\u00e9"})", + R"([{"a": [1, 2, {"b": "c"}]}, [], {}, "\ud83d\ude00", 1e10])", + "{\"\xC3\xA9\": \"\xF0\x9F\x98\x80\"}", + R"( {"k" : [ "v" , 0 ] } )", + }; + // each character that can be inserted, including a null byte + const std::string insertions("[]{},:\"x\\\0\xFF", 11); + + std::vector inputs; + for (const auto& doc : documents) + { + for (std::size_t i = 0; i <= doc.size(); ++i) + { + inputs.push_back(doc.substr(0, i)); + if (i < doc.size()) + { + inputs.push_back(doc.substr(0, i) + doc.substr(i + 1)); + } + for (const char c : insertions) + { + inputs.push_back(doc.substr(0, i) + c + doc.substr(i)); + } + } + } + + for (const auto& s : inputs) + { + CAPTURE(s); + const auto result = parse_recovering(s); + CHECK(result.balanced); + CHECK(result.errors.size() <= s.size() + 1); + CHECK(result.events <= (4 * s.size()) + 4); + if (json::accept(s)) + { + CHECK(result.ok); + CHECK(result.errors.empty()); + CHECK(result.value == json::parse(s)); + } + else + { + CHECK(!result.ok); + CHECK(!result.errors.empty()); + } + } + } +} diff --git a/tests/src/unit-regression2.cpp b/tests/src/unit-regression2.cpp index c466958bf..193464fca 100644 --- a/tests/src/unit-regression2.cpp +++ b/tests/src/unit-regression2.cpp @@ -870,4 +870,262 @@ TEST_CASE("regression test - excessive binary container size honors allow_except CHECK(json::from_cbor(std::vector {0x9b, 0, 0, 0, 0, 0, 0, 0, 0x02}, true, false).is_discarded()); } +namespace +{ +/// builds a value from SAX events, asks the parser to recover from its first +/// 100 errors, and checks that the events are balanced (see #3989) +class RecoveringParser : public nlohmann::detail::json_sax_dom_parser +{ + using base = nlohmann::detail::json_sax_dom_parser; + + public: + explicit RecoveringParser(json& j) + : base(j, false) + {} + + bool null() + { + value(); + return base::null(); + } + + bool boolean(bool val) + { + value(); + return base::boolean(val); + } + + bool number_integer(json::number_integer_t val) + { + value(); + return base::number_integer(val); + } + + bool number_unsigned(json::number_unsigned_t val) + { + value(); + return base::number_unsigned(val); + } + + bool number_float(json::number_float_t val, const std::string& s) + { + value(); + return base::number_float(val, s); + } + + bool string(std::string& val) + { + value(); + return base::string(val); + } + + bool binary(json::binary_t& val) + { + value(); + return base::binary(val); + } + + bool start_object(std::size_t elements) + { + value(); + stack.push_back('o'); + return base::start_object(elements); + } + + bool key(std::string& val) + { + if (stack.empty() || stack.back() != 'o') + { + well_formed = false; + return false; + } + stack.back() = 'v'; + return base::key(val); + } + + bool end_object() + { + if (stack.empty() || stack.back() != 'o') + { + well_formed = false; + return false; + } + stack.pop_back(); + return base::end_object(); + } + + bool start_array(std::size_t elements) + { + value(); + stack.push_back('a'); + return base::start_array(elements); + } + + bool end_array() + { + if (stack.empty() || stack.back() != 'a') + { + well_formed = false; + return false; + } + stack.pop_back(); + return base::end_array(); + } + + bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, const json::exception& /*unused*/) + { + // a limit, so that a reader that does not stop fails the test + // instead of making it hang + return ++errors < 100; + } + + /// whether the events were balanced and every key was followed by a value + bool balanced() const + { + return well_formed && stack.empty(); + } + + std::size_t errors = 0; + std::vector stack {}; // NOLINT(readability-redundant-member-init) + bool well_formed = true; + + private: + void value() + { + if (!stack.empty()) + { + if (stack.back() == 'v') + { + stack.back() = 'o'; + } + else if (stack.back() == 'o') + { + well_formed = false; + } + } + } + +}; + +struct BinaryParseResult +{ + json value; + std::size_t errors; + bool ok; + bool balanced; +}; + +BinaryParseResult parse_binary_recovering(const std::vector& input, const json::input_format_t format) +{ + json j; + RecoveringParser sax(j); + const bool ok = json::sax_parse(input, &sax, format); + return {j, sax.errors, ok, sax.balanced()}; +} +} // namespace + +TEST_CASE("regression test - #3989 SAX parse_error() returning true") +{ + SECTION("binary formats stop after an error and complete what was read") + { + const json j = {{"a", {1, -2, {{"b", "c"}}, json::array()}}, {"d", {{"e", nullptr}, {"f", true}}}, {"g", 1.5}, {"h", json::binary({1, 2, 3})}}; + + const std::vector>> encodings = + { + {json::input_format_t::cbor, json::to_cbor(j)}, + {json::input_format_t::msgpack, json::to_msgpack(j)}, + {json::input_format_t::ubjson, json::to_ubjson(j)}, + {json::input_format_t::ubjson, json::to_ubjson(j, true, true)}, + {json::input_format_t::bjdata, json::to_bjdata(j)}, + {json::input_format_t::bjdata, json::to_bjdata(j, true, true)}, + {json::input_format_t::bson, json::to_bson(j)}, + {json::input_format_t::bon8, json::to_bon8(j)}, + }; + + for (const auto& encoding : encodings) + { + const auto format = encoding.first; + const auto& bytes = encoding.second; + CAPTURE(format); + + // every prefix is truncated input + for (std::size_t length = 0; length < bytes.size(); ++length) + { + CAPTURE(length); + const auto result = parse_binary_recovering(std::vector(bytes.begin(), bytes.begin() + static_cast(length)), format); + CHECK(!result.ok); + CHECK(result.errors == 1); + CHECK(result.balanced); + } + + // the complete input is read as usual (binary values do not + // round-trip through every format, so compare with a plain parse) + json expected; + nlohmann::detail::json_sax_dom_parser dom(expected); + CHECK(json::sax_parse(bytes, &dom, format)); + const auto complete = parse_binary_recovering(bytes, format); + CHECK(complete.ok); + CHECK(complete.errors == 0); + CHECK(complete.value == expected); + + // a byte after the value + auto trailing_bytes = bytes; + trailing_bytes.push_back(0x01); + const auto trailing = parse_binary_recovering(trailing_bytes, format); + CHECK(!trailing.ok); + CHECK(trailing.errors == 1); + CHECK(trailing.value == expected); + } + } + + SECTION("containers without an end") + { + // these made the readers loop, or read on, after the error + const auto cbor_array = parse_binary_recovering({0x9F}, json::input_format_t::cbor); + CHECK(cbor_array.errors == 1); + CHECK(cbor_array.value == json::array()); + + const auto cbor_map = parse_binary_recovering({0xBF, 0x61, 'a'}, json::input_format_t::cbor); + CHECK(cbor_map.errors == 1); + CHECK(cbor_map.value == json({{"a", nullptr}})); + + const auto msgpack_array = parse_binary_recovering({0xDD, 0xFF, 0xFF, 0xFF, 0xFF}, json::input_format_t::msgpack); + CHECK(msgpack_array.errors == 1); + CHECK(msgpack_array.value == json::array()); + + const auto msgpack_map = parse_binary_recovering({0x81, 0xA1, 'a', 0x92, 0x01}, json::input_format_t::msgpack); + CHECK(msgpack_map.errors == 1); + CHECK(msgpack_map.value == json({{"a", {1}}})); + } + + SECTION("BJData ndarray") + { + // a 2x3 int8 array with two of its six elements; the annotated array + // format opens an object and two arrays of its own + const auto result = parse_binary_recovering({'[', '$', 'i', '#', '[', '$', 'i', '#', 'i', 2, 2, 3, 1, 2}, json::input_format_t::bjdata); + CHECK(result.errors == 1); + CHECK(result.balanced); + CHECK(result.value == json({{"_ArrayType_", "int8"}, {"_ArraySize_", {2, 3}}, {"_ArrayData_", {1, 2}}})); + } + + SECTION("JSON text") + { + // the parser stopped, but reported success + json j; + RecoveringParser sax(j); + CHECK(!json::sax_parse("[1,2,3,]", &sax)); + CHECK(sax.errors == 1); + CHECK(j == json({1, 2, 3})); + } + + SECTION("the SAX parsers of the library stop") + { + json _; + CHECK(json::from_cbor(std::vector {0x9F}, true, false).is_discarded()); + CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector {0x9F}), "[json.exception.parse_error.110] parse error at byte 2: syntax error while parsing CBOR value: unexpected end of input", json::parse_error&); + CHECK(json::parse("[1,2,3,]", nullptr, false).is_discarded()); + CHECK(!json::accept("[1,2,3,]")); + } +} + DOCTEST_CLANG_SUPPRESS_WARNING_POP