mirror of
https://github.com/nlohmann/json.git
synced 2026-10-03 05:00:30 +00:00
Repair complete items in binary formats when parse_error() returns true (#3989)
When the SAX parser asks to recover, the binary readers now repair an item whose end is known and read on after it, as RFC 8949, Section 5.3 describes for CBOR: - CBOR: tags are ignored, and simple values other than false, true, and null become null (RFC 8949, Section 6.1); a negative integer below the range of number_integer_t becomes the nearest floating-point number. - Strings that are not valid UTF-8 get U+FFFD for each ill-formed sequence, as in JSON text; so does a UBJSON/BJData char above 0x7F. - UBJSON/BJData high-precision numbers keep their longest valid beginning (via the lexer's recover_token()), or become infinity. - Members whose key is not a string are skipped (CBOR, MessagePack, BON8), like members without a key in JSON text. - BSON elements of types the library does not read (ObjectId, datetime, decimal128, ...) become null; a string without its terminator and a document whose size does not match are kept. Where the end of an item is unknown, reading stops as before, except that BSON skips to the end of the document, whose size it knows. The value read before such an error is now completed by the reader from its container stack, as the JSON parser does, instead of by a proxy SAX parser, which is removed. Like the parser, binary_reader gets an AllowRecovery template parameter, so that from_*() compile without the new code. Tests: a table of repairs, numbers out of range, errors that stop, and a sweep over changed and removed bytes of eight encodings that checks balanced events and that the first error is the one from_*() reports. All fuzzers now run a recovering checker; the binary ones also check that it reports an error exactly when from_*() fails. Signed-off-by: Niels Lohmann <mail@nlohmann.me>
This commit is contained in:
Niels Lohmann
File diff suppressed because it is too large
Load Diff
@@ -132,8 +132,8 @@ struct json_sax
|
||||
@param[in] last_token the last read token
|
||||
@param[in] ex an exception object describing the error
|
||||
@return whether to recover from the error: false stops parsing; true
|
||||
repairs JSON text and continues, or, for the binary formats, stops
|
||||
after closing the containers read so far
|
||||
repairs the error and continues, or, if that is not possible,
|
||||
stops after completing the value read so far
|
||||
*/
|
||||
virtual bool parse_error(std::size_t position,
|
||||
const std::string& last_token,
|
||||
@@ -1212,176 +1212,5 @@ class json_sax_acceptor
|
||||
}
|
||||
};
|
||||
|
||||
/*!
|
||||
@brief SAX proxy that lets the binary readers keep what was read before an error
|
||||
|
||||
The binary formats cannot continue after an error: a value's size is given
|
||||
before its payload, and every byte value is a valid type marker, so there is no
|
||||
way to find where the next value begins. When the SAX parser's parse_error()
|
||||
returns true to ask for error recovery, the best the binary readers can offer is
|
||||
the value read up to the error.
|
||||
|
||||
This proxy forwards every event to the SAX parser and records which containers
|
||||
are open and whether a key still waits for its value. After an error the SAX
|
||||
parser asked to recover from, @ref close_open_containers then completes the
|
||||
value with null for a pending key and the missing end events, so the SAX parser
|
||||
sees balanced events (see #3989).
|
||||
|
||||
@tparam BasicJsonType the JSON type
|
||||
@tparam SAX the SAX parser to forward the events to
|
||||
*/
|
||||
template<typename BasicJsonType, typename SAX>
|
||||
class json_sax_salvager
|
||||
{
|
||||
public:
|
||||
using number_integer_t = typename BasicJsonType::number_integer_t;
|
||||
using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
|
||||
using number_float_t = typename BasicJsonType::number_float_t;
|
||||
using string_t = typename BasicJsonType::string_t;
|
||||
using binary_t = typename BasicJsonType::binary_t;
|
||||
|
||||
explicit json_sax_salvager(SAX* sax_) noexcept
|
||||
: sax(sax_)
|
||||
{}
|
||||
|
||||
bool null()
|
||||
{
|
||||
key_pending = false;
|
||||
return sax->null();
|
||||
}
|
||||
|
||||
bool boolean(bool val)
|
||||
{
|
||||
key_pending = false;
|
||||
return sax->boolean(val);
|
||||
}
|
||||
|
||||
bool number_integer(number_integer_t val)
|
||||
{
|
||||
key_pending = false;
|
||||
return sax->number_integer(val);
|
||||
}
|
||||
|
||||
bool number_unsigned(number_unsigned_t val)
|
||||
{
|
||||
key_pending = false;
|
||||
return sax->number_unsigned(val);
|
||||
}
|
||||
|
||||
bool number_float(number_float_t val, const string_t& s)
|
||||
{
|
||||
key_pending = false;
|
||||
return sax->number_float(val, s);
|
||||
}
|
||||
|
||||
bool string(string_t& val)
|
||||
{
|
||||
key_pending = false;
|
||||
return sax->string(val);
|
||||
}
|
||||
|
||||
bool binary(binary_t& val)
|
||||
{
|
||||
key_pending = false;
|
||||
return sax->binary(val);
|
||||
}
|
||||
|
||||
bool start_object(std::size_t len)
|
||||
{
|
||||
key_pending = false;
|
||||
if (JSON_HEDLEY_UNLIKELY(!sax->start_object(len)))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
open_containers.push_back(true);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool key(string_t& val)
|
||||
{
|
||||
key_pending = true;
|
||||
return sax->key(val);
|
||||
}
|
||||
|
||||
bool end_object()
|
||||
{
|
||||
JSON_ASSERT(!open_containers.empty() && open_containers.back());
|
||||
open_containers.pop_back();
|
||||
return sax->end_object();
|
||||
}
|
||||
|
||||
bool start_array(std::size_t len)
|
||||
{
|
||||
key_pending = false;
|
||||
if (JSON_HEDLEY_UNLIKELY(!sax->start_array(len)))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
open_containers.push_back(false);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool end_array()
|
||||
{
|
||||
JSON_ASSERT(!open_containers.empty() && !open_containers.back());
|
||||
open_containers.pop_back();
|
||||
return sax->end_array();
|
||||
}
|
||||
|
||||
template<class Exception>
|
||||
bool parse_error(std::size_t position, const std::string& last_token,
|
||||
const Exception& ex)
|
||||
{
|
||||
recovery_requested = sax->parse_error(position, last_token, ex);
|
||||
// the binary readers stop after an error anyway
|
||||
return false;
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief complete the value read before an error
|
||||
|
||||
Does nothing unless the SAX parser's parse_error() returned true. Otherwise
|
||||
passes null for a key that waits for its value and closes the containers
|
||||
that are still open, innermost first, until an event returns false.
|
||||
*/
|
||||
void close_open_containers()
|
||||
{
|
||||
if (!recovery_requested)
|
||||
{
|
||||
return;
|
||||
}
|
||||
recovery_requested = false;
|
||||
|
||||
if (key_pending)
|
||||
{
|
||||
key_pending = false;
|
||||
if (JSON_HEDLEY_UNLIKELY(!sax->null()))
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
while (!open_containers.empty())
|
||||
{
|
||||
const bool is_object = open_containers.back();
|
||||
open_containers.pop_back();
|
||||
if (JSON_HEDLEY_UNLIKELY(is_object ? !sax->end_object() : !sax->end_array()))
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
/// the SAX parser the events are forwarded to
|
||||
SAX* sax = nullptr;
|
||||
/// the containers that are open, innermost last; true for an object
|
||||
std::vector<bool> open_containers {}; // NOLINT(readability-redundant-member-init)
|
||||
/// whether a key was passed whose value has not been passed yet
|
||||
bool key_pending = false;
|
||||
/// whether the SAX parser's parse_error() asked to recover from the error
|
||||
bool recovery_requested = false;
|
||||
};
|
||||
|
||||
} // namespace detail
|
||||
NLOHMANN_JSON_NAMESPACE_END
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
#include <cstddef> // size_t
|
||||
#include <cstdint> // uint8_t, uint32_t
|
||||
#include <string> // string, to_string
|
||||
#include <utility> // move
|
||||
|
||||
#include <nlohmann/detail/abi_macros.hpp>
|
||||
#include <nlohmann/detail/macro_scope.hpp>
|
||||
@@ -133,5 +134,78 @@ inline bool is_valid_utf8(const StringType& s, const std::size_t first = 0) noex
|
||||
return state == UTF8_ACCEPT;
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief append U+FFFD REPLACEMENT CHARACTER, encoded in UTF-8
|
||||
@param[in,out] s the string to append to
|
||||
*/
|
||||
template<typename StringType>
|
||||
inline void append_replacement_character(StringType& s)
|
||||
{
|
||||
s.push_back(static_cast<typename StringType::value_type>(0xEFu));
|
||||
s.push_back(static_cast<typename StringType::value_type>(0xBFu));
|
||||
s.push_back(static_cast<typename StringType::value_type>(0xBDu));
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief replace ill-formed UTF-8 with U+FFFD REPLACEMENT CHARACTER
|
||||
|
||||
Each maximal subpart of an ill-formed sequence becomes one U+FFFD, as the
|
||||
Unicode Standard recommends (Section 3.9, "U+FFFD Substitution of Maximal
|
||||
Subparts"), and as the parser for JSON text does when it recovers from errors.
|
||||
|
||||
@param[in,out] s the string to repair
|
||||
@param[in] first index of the first byte to repair; the bytes before it are
|
||||
assumed to be valid UTF-8 that ends on a code point boundary
|
||||
*/
|
||||
template<typename StringType>
|
||||
inline void replace_invalid_utf8(StringType& s, const std::size_t first = 0)
|
||||
{
|
||||
StringType result = s;
|
||||
result.resize(first);
|
||||
|
||||
std::uint8_t state = UTF8_ACCEPT;
|
||||
std::uint32_t codepoint = 0;
|
||||
// the first byte of the sequence being decoded
|
||||
std::size_t sequence_start = first;
|
||||
|
||||
std::size_t i = first;
|
||||
while (i < s.size())
|
||||
{
|
||||
switch (decode(state, codepoint, static_cast<std::uint8_t>(s[i])))
|
||||
{
|
||||
case UTF8_ACCEPT:
|
||||
for (++i; sequence_start < i; ++sequence_start)
|
||||
{
|
||||
result.push_back(s[sequence_start]);
|
||||
}
|
||||
break;
|
||||
|
||||
case UTF8_REJECT:
|
||||
append_replacement_character(result);
|
||||
// the byte that made the sequence ill-formed begins the next
|
||||
// one, unless it began this one
|
||||
if (i == sequence_start)
|
||||
{
|
||||
++i;
|
||||
}
|
||||
state = UTF8_ACCEPT;
|
||||
sequence_start = i;
|
||||
break;
|
||||
|
||||
default: // in the middle of a sequence
|
||||
++i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// a sequence that the string ends in the middle of
|
||||
if (state != UTF8_ACCEPT)
|
||||
{
|
||||
append_replacement_character(result);
|
||||
}
|
||||
|
||||
s = std::move(result);
|
||||
}
|
||||
|
||||
} // namespace detail
|
||||
NLOHMANN_JSON_NAMESPACE_END
|
||||
|
||||
@@ -143,7 +143,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
|
||||
friend class ::nlohmann::detail::iter_impl;
|
||||
template<typename BasicJsonType, typename CharType, typename OutputSinkType>
|
||||
friend class ::nlohmann::detail::binary_writer;
|
||||
template<typename BasicJsonType, typename InputType, typename SAX>
|
||||
template<typename BasicJsonType, typename InputType, typename SAX, bool AllowRecovery>
|
||||
friend class ::nlohmann::detail::binary_reader;
|
||||
template<typename BasicJsonType, typename InputAdapterType>
|
||||
friend class ::nlohmann::detail::json_sax_dom_parser;
|
||||
@@ -4979,26 +4979,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
|
||||
return parser(i.get(), nullptr, false, ignore_comments, ignore_trailing_commas, true).accept(true);
|
||||
}
|
||||
|
||||
private:
|
||||
/// read a binary format and pass it to a SAX parser; if the SAX parser
|
||||
/// asks to recover from an error, the value read so far is completed
|
||||
/// (see detail::json_sax_salvager and #3989)
|
||||
template<typename InputAdapterType, typename SAX>
|
||||
static bool sax_parse_binary(InputAdapterType ia, SAX* sax,
|
||||
const input_format_t format, const bool strict)
|
||||
{
|
||||
(void)detail::is_sax_static_asserts<SAX, basic_json> {};
|
||||
using salvager_t = detail::json_sax_salvager<basic_json, SAX>;
|
||||
salvager_t salvager(sax);
|
||||
const bool result = detail::binary_reader<basic_json, InputAdapterType, salvager_t>(std::move(ia), format).sax_parse(format, &salvager, strict);
|
||||
if (!result)
|
||||
{
|
||||
salvager.close_open_containers();
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public:
|
||||
/// @brief generate SAX events
|
||||
/// @sa https://json.nlohmann.me/api/basic_json/sax_parse/
|
||||
template <typename InputType, typename SAX>
|
||||
@@ -5012,7 +4992,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
|
||||
auto ia = detail::input_adapter(std::forward<InputType>(i));
|
||||
return format == input_format_t::json
|
||||
? parser(std::move(ia), nullptr, true, ignore_comments, ignore_trailing_commas).sax_parse(sax, strict)
|
||||
: sax_parse_binary(std::move(ia), sax, format, strict);
|
||||
: detail::binary_reader<basic_json, decltype(ia), SAX, true>(std::move(ia), format).sax_parse(format, sax, strict);
|
||||
}
|
||||
|
||||
/// @brief generate SAX events (iterator pair, or iterator+sentinel pair for C++20 ranges support)
|
||||
@@ -5029,7 +5009,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
|
||||
auto ia = detail::input_adapter(std::move(first), std::move(last));
|
||||
return format == input_format_t::json
|
||||
? parser(std::move(ia), nullptr, true, ignore_comments, ignore_trailing_commas).sax_parse(sax, strict)
|
||||
: sax_parse_binary(std::move(ia), sax, format, strict);
|
||||
: detail::binary_reader<basic_json, decltype(ia), SAX, true>(std::move(ia), format).sax_parse(format, sax, strict);
|
||||
}
|
||||
|
||||
/// @brief generate SAX events
|
||||
@@ -5051,7 +5031,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
|
||||
// NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg)
|
||||
? parser(std::move(ia), nullptr, true, ignore_comments, ignore_trailing_commas).sax_parse(sax, strict)
|
||||
// NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg)
|
||||
: sax_parse_binary(std::move(ia), sax, format, strict);
|
||||
: detail::binary_reader<basic_json, decltype(ia), SAX, true>(std::move(ia), format).sax_parse(format, sax, strict);
|
||||
}
|
||||
#ifndef JSON_NO_IO
|
||||
/// @brief deserialize from stream
|
||||
|
||||
Reference in New Issue
Block a user