mirror of
https://github.com/nlohmann/json.git
synced 2026-10-16 11:17:13 +00:00
Merge remote-tracking branch 'origin/develop' into claude/fix-issue-3989-db7e45
Signed-off-by: Niels Lohmann <mail@nlohmann.me>
This commit is contained in:
124 files changed
+13231
-2994
No files matched your search
@@ -223,7 +223,7 @@ class binary_reader
|
||||
|
||||
if (JSON_HEDLEY_UNLIKELY(current != char_traits<char_type>::eof()))
|
||||
{
|
||||
return last_byte_error(exception_id::unexpected_end_of_input, concat("expected end of input; last byte: 0x", get_token_string()), "value");
|
||||
return last_byte_error(exception_id::unexpected_end_of_input, "expected end of input", "value", "");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -354,7 +354,7 @@ class binary_reader
|
||||
{
|
||||
if (JSON_HEDLEY_UNLIKELY(document_size < 0 || static_cast<std::size_t>(document_size) != chars_read - document_start))
|
||||
{
|
||||
static_cast<void>(last_byte_error(exception_id::unexpected_byte, concat("document size ", std::to_string(document_size), " does not match the number of bytes read (", std::to_string(chars_read - document_start), ")"), "document"));
|
||||
static_cast<void>(bson_document_size_error(document_size, chars_read - document_start));
|
||||
return repair_requested();
|
||||
}
|
||||
return true;
|
||||
@@ -678,7 +678,7 @@ class binary_reader
|
||||
{
|
||||
// the end of the element is lost (see resync)
|
||||
skip_requested = true;
|
||||
return last_byte_error(exception_id::unexpected_byte, concat("string length must be at least 1, is ", std::to_string(len)), "string");
|
||||
return bson_length_error("string length must be at least 1, is ", len, "string");
|
||||
}
|
||||
|
||||
if (JSON_HEDLEY_UNLIKELY(!get_string(len - static_cast<NumberType>(1), result)))
|
||||
@@ -713,7 +713,7 @@ class binary_reader
|
||||
{
|
||||
// the end of the element is lost (see resync)
|
||||
skip_requested = true;
|
||||
return last_byte_error(exception_id::unexpected_byte, concat("byte array length cannot be negative, is ", std::to_string(len)), "binary");
|
||||
return bson_length_error("byte array length cannot be negative, is ", len, "binary");
|
||||
}
|
||||
|
||||
// All BSON binary values have a subtype
|
||||
@@ -803,9 +803,7 @@ class binary_reader
|
||||
|
||||
default: // anything else is not supported (yet)
|
||||
{
|
||||
const std::string cr_str = hex_byte(static_cast<std::uint8_t>(element_type));
|
||||
static_cast<void>(report_error(element_type_parse_position, cr_str,
|
||||
parse_error::create(exception_id::bson_unsupported_type, element_type_parse_position, concat("Unsupported BSON record type 0x", cr_str), nullptr)));
|
||||
static_cast<void>(bson_unsupported_type_error(element_type_parse_position, element_type));
|
||||
return skip_unsupported_bson_element(element_type);
|
||||
}
|
||||
}
|
||||
@@ -1259,6 +1257,7 @@ class binary_reader
|
||||
@param[in] context parsing context for the error message
|
||||
@return false, so that the caller stops reading
|
||||
*/
|
||||
JSON_HEDLEY_NEVER_INLINE
|
||||
bool cbor_indefinite_string_error(const char* type_name, const char* context)
|
||||
{
|
||||
return last_byte_error(exception_id::invalid_string_or_size, concat("indefinite-length ", type_name, " is not allowed inside indefinite-length ", type_name, "; last byte: 0x", get_token_string()), context);
|
||||
@@ -1338,7 +1337,7 @@ class binary_reader
|
||||
|
||||
default:
|
||||
{
|
||||
return last_byte_error(exception_id::invalid_string_or_size, concat("expected length specification (0x60-0x7B)", inside_indefinite ? "" : " or indefinite string type (0x7F)", "; last byte: 0x", get_token_string()), "string");
|
||||
return last_byte_error(exception_id::invalid_string_or_size, "expected length specification (0x60-0x7B)", "string", inside_indefinite ? "" : " or indefinite string type (0x7F)");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1472,7 +1471,7 @@ class binary_reader
|
||||
// a key that begins an item is skipped with its value when recovering
|
||||
// (see resync); a break stop code or a reserved byte begins none
|
||||
skip_requested = is_cbor_item_head(static_cast<unsigned int>(current));
|
||||
return last_byte_error(exception_id::invalid_string_or_size, concat("only string keys are supported, but found ", found, "; last byte: 0x", get_token_string()), "object key");
|
||||
return last_byte_error(exception_id::invalid_string_or_size, "only string keys are supported, but found ", "object key", found);
|
||||
}
|
||||
|
||||
/*!
|
||||
@@ -1553,7 +1552,7 @@ class binary_reader
|
||||
|
||||
default:
|
||||
{
|
||||
return last_byte_error(exception_id::invalid_string_or_size, concat("expected length specification (0x40-0x5B)", inside_indefinite ? "" : " or indefinite binary array type (0x5F)", "; last byte: 0x", get_token_string()), "binary");
|
||||
return last_byte_error(exception_id::invalid_string_or_size, "expected length specification (0x40-0x5B)", "binary", inside_indefinite ? "" : " or indefinite binary array type (0x5F)");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1699,8 +1698,7 @@ class binary_reader
|
||||
{
|
||||
if (JSON_HEDLEY_UNLIKELY(!value_in_range_of<std::size_t>(len) || len == detail::unknown_size()))
|
||||
{
|
||||
return report_error(chars_read, get_token_string(), out_of_range::create(exception_id::container_too_large,
|
||||
exception_message(concat("excessive ", context, " size"), "size"), nullptr));
|
||||
return container_size_error(context);
|
||||
}
|
||||
result = conditional_static_cast<std::size_t>(len);
|
||||
return true;
|
||||
@@ -2388,7 +2386,7 @@ class binary_reader
|
||||
|
||||
default:
|
||||
{
|
||||
return last_byte_error(exception_id::invalid_string_or_size, concat("expected length specification (0xA0-0xBF, 0xD9-0xDB); last byte: 0x", get_token_string()), "string");
|
||||
return last_byte_error(exception_id::invalid_string_or_size, "expected length specification (0xA0-0xBF, 0xD9-0xDB)", "string", "");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2483,7 +2481,7 @@ class binary_reader
|
||||
|
||||
// the key is skipped with its value when recovering (see resync)
|
||||
skip_requested = true;
|
||||
return last_byte_error(exception_id::invalid_string_or_size, concat("only string keys are supported, but found ", found, "; last byte: 0x", get_token_string()), "object key");
|
||||
return last_byte_error(exception_id::invalid_string_or_size, "only string keys are supported, but found ", "object key", found);
|
||||
}
|
||||
|
||||
/*!
|
||||
@@ -3128,8 +3126,7 @@ class binary_reader
|
||||
}
|
||||
if (JSON_HEDLEY_UNLIKELY(!value_in_range_of<std::size_t>(number)))
|
||||
{
|
||||
return report_error(chars_read, get_token_string(), out_of_range::create(exception_id::container_too_large,
|
||||
exception_message("integer value overflow", "size"), nullptr));
|
||||
return out_of_range_error(exception_id::container_too_large, "integer value overflow", "size");
|
||||
}
|
||||
result = static_cast<std::size_t>(number); // NOLINT(bugprone-signed-char-misuse,cert-str34-c): number is not a char
|
||||
return true;
|
||||
@@ -3226,8 +3223,7 @@ class binary_reader
|
||||
}
|
||||
if (!value_in_range_of<std::size_t>(number))
|
||||
{
|
||||
return report_error(chars_read, get_token_string(), out_of_range::create(exception_id::container_too_large,
|
||||
exception_message("integer value overflow", "size"), nullptr));
|
||||
return out_of_range_error(exception_id::container_too_large, "integer value overflow", "size");
|
||||
}
|
||||
result = detail::conditional_static_cast<std::size_t>(number);
|
||||
return true;
|
||||
@@ -3308,7 +3304,7 @@ class binary_reader
|
||||
// or SIZE_MAX.
|
||||
if (JSON_HEDLEY_UNLIKELY(result > (std::numeric_limits<std::size_t>::max)() / i))
|
||||
{
|
||||
return report_error(chars_read, get_token_string(), out_of_range::create(exception_id::container_too_large, exception_message("excessive ndarray size caused overflow", "size"), nullptr));
|
||||
return out_of_range_error(exception_id::container_too_large, "excessive ndarray size caused overflow", "size");
|
||||
}
|
||||
result *= i;
|
||||
// the pre-check above already rules out result becoming 0
|
||||
@@ -3317,7 +3313,7 @@ class binary_reader
|
||||
// unknown-size container (see get_ubjson_size_type())
|
||||
if (result == npos)
|
||||
{
|
||||
return report_error(chars_read, get_token_string(), out_of_range::create(exception_id::container_too_large, exception_message("excessive ndarray size caused overflow", "size"), nullptr));
|
||||
return out_of_range_error(exception_id::container_too_large, "excessive ndarray size caused overflow", "size");
|
||||
}
|
||||
if (JSON_HEDLEY_UNLIKELY(!emit_unsigned(i)))
|
||||
{
|
||||
@@ -3368,7 +3364,7 @@ class binary_reader
|
||||
if (input_format == input_format_t::bjdata
|
||||
&& JSON_HEDLEY_UNLIKELY(is_bjd_excluded_optimized_type(result.second)))
|
||||
{
|
||||
return last_byte_error(exception_id::unexpected_byte, concat("marker 0x", get_token_string(), " is not a permitted optimized array type"), "type");
|
||||
return optimized_type_error();
|
||||
}
|
||||
|
||||
if (JSON_HEDLEY_UNLIKELY(!unexpect_eof("type")))
|
||||
@@ -3383,7 +3379,7 @@ class binary_reader
|
||||
{
|
||||
return false;
|
||||
}
|
||||
return last_byte_error(exception_id::unexpected_byte, concat("expected '#' after type information; last byte: 0x", get_token_string()), "size");
|
||||
return last_byte_error(exception_id::unexpected_byte, "expected '#' after type information", "size", "");
|
||||
}
|
||||
|
||||
const bool is_error = get_ubjson_size_value(result.first, is_ndarray, 0, result.second);
|
||||
@@ -3534,7 +3530,7 @@ class binary_reader
|
||||
}
|
||||
if (JSON_HEDLEY_UNLIKELY(current > 127))
|
||||
{
|
||||
static_cast<void>(last_byte_error(exception_id::invalid_string_or_size, concat("byte after 'C' must be in range 0x00..0x7F; last byte: 0x", get_token_string()), "char"));
|
||||
static_cast<void>(last_byte_error(exception_id::invalid_string_or_size, "byte after 'C' must be in range 0x00..0x7F", "char", ""));
|
||||
if (!repair_requested())
|
||||
{
|
||||
return false;
|
||||
@@ -3628,8 +3624,7 @@ class binary_reader
|
||||
if (JSON_HEDLEY_UNLIKELY((size_and_type.second == 'Z' || size_and_type.second == 'T' || size_and_type.second == 'F')
|
||||
&& size_and_type.first > max_valueless_container_size))
|
||||
{
|
||||
return report_error(chars_read, get_token_string(), out_of_range::create(exception_id::container_too_large,
|
||||
exception_message("excessive array size", "size"), nullptr));
|
||||
return out_of_range_error(exception_id::container_too_large, "excessive array size", "size");
|
||||
}
|
||||
|
||||
if (JSON_HEDLEY_UNLIKELY(!enter_array(size_and_type.first, size_and_type.second)))
|
||||
@@ -4540,7 +4535,8 @@ class binary_reader
|
||||
{
|
||||
// in case of failure, advance position by 1 to report the failing location
|
||||
++chars_read;
|
||||
return report_error(chars_read, "<end of file>", parse_error::create(exception_id::unexpected_end_of_input, chars_read, exception_message("unexpected end of input", context), nullptr));
|
||||
static_cast<void>(report_eof(context));
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
@@ -4692,8 +4688,7 @@ class binary_reader
|
||||
const auto result = static_cast<number_float_t>(number);
|
||||
if (JSON_HEDLEY_UNLIKELY(std::isfinite(number) && !std::isfinite(result)))
|
||||
{
|
||||
static_cast<void>(report_error(chars_read, get_token_string(),
|
||||
out_of_range::create(exception_id::number_overflow, exception_message("number overflow", "value"), nullptr)));
|
||||
static_cast<void>(out_of_range_error(exception_id::number_overflow, "number overflow", "value"));
|
||||
// when recovering, the number becomes infinity, as in JSON text
|
||||
if (!repair_requested())
|
||||
{
|
||||
@@ -5213,22 +5208,82 @@ class binary_reader
|
||||
{
|
||||
if (JSON_HEDLEY_UNLIKELY(current == char_traits<char_type>::eof()))
|
||||
{
|
||||
return report_error(chars_read, "<end of file>",
|
||||
parse_error::create(exception_id::unexpected_end_of_input, chars_read, exception_message("unexpected end of input", context), nullptr));
|
||||
return report_eof(context);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief reports an unexpected end of input (out of line to keep the exception
|
||||
construction out of the hot reading functions)
|
||||
@param[in] context further context information (for diagnostics)
|
||||
@return false, so that the caller stops reading (see report_error)
|
||||
*/
|
||||
JSON_HEDLEY_NON_NULL(2)
|
||||
JSON_HEDLEY_NEVER_INLINE
|
||||
bool report_eof(const char* context)
|
||||
{
|
||||
return report_error(chars_read, "<end of file>",
|
||||
parse_error::create(exception_id::unexpected_end_of_input, chars_read, exception_message("unexpected end of input", context), nullptr));
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief reports an out_of_range error at the last read byte (out of line)
|
||||
*/
|
||||
JSON_HEDLEY_NEVER_INLINE
|
||||
bool out_of_range_error(const exception_id id_, const char* detail, const char* context)
|
||||
{
|
||||
return report_error(chars_read, get_token_string(), out_of_range::create(id_, exception_message(detail, context), nullptr));
|
||||
}
|
||||
|
||||
JSON_HEDLEY_NEVER_INLINE
|
||||
bool container_size_error(const char* what)
|
||||
{
|
||||
return report_error(chars_read, get_token_string(), out_of_range::create(exception_id::container_too_large,
|
||||
exception_message(concat("excessive ", what, " size"), "size"), nullptr));
|
||||
}
|
||||
|
||||
JSON_HEDLEY_NEVER_INLINE
|
||||
bool optimized_type_error()
|
||||
{
|
||||
return last_byte_error(exception_id::unexpected_byte, concat("marker 0x", get_token_string(), " is not a permitted optimized array type"), "type");
|
||||
}
|
||||
|
||||
template<typename NumberType>
|
||||
JSON_HEDLEY_NEVER_INLINE
|
||||
bool bson_length_error(const char* prefix, const NumberType len, const char* context)
|
||||
{
|
||||
return last_byte_error(exception_id::unexpected_byte, concat(prefix, std::to_string(len)), context);
|
||||
}
|
||||
|
||||
JSON_HEDLEY_NEVER_INLINE
|
||||
bool bson_document_size_error(const std::int32_t document_size, const std::size_t bytes_read)
|
||||
{
|
||||
return last_byte_error(exception_id::unexpected_byte, concat("document size ", std::to_string(document_size), " does not match the number of bytes read (", std::to_string(bytes_read), ")"), "document");
|
||||
}
|
||||
|
||||
JSON_HEDLEY_NEVER_INLINE
|
||||
bool bson_unsupported_type_error(const std::size_t position, const char element_type)
|
||||
{
|
||||
const std::string cr_str = hex_byte(static_cast<std::uint8_t>(element_type));
|
||||
return report_error(position, cr_str,
|
||||
parse_error::create(exception_id::bson_unsupported_type, position, concat("Unsupported BSON record type 0x", cr_str), nullptr));
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief reports a parse error at the last read byte
|
||||
@param[in] id_ the id of the parse_error exception
|
||||
@param[in] detail a detailed error message
|
||||
@param[in] context further context information
|
||||
@param[in] tail optional text appended to the detail, followed by "; last byte: 0x<token>"
|
||||
@return false, so that the caller stops reading (see report_error)
|
||||
*/
|
||||
bool last_byte_error(const exception_id id_, const std::string& detail, const std::string& context)
|
||||
JSON_HEDLEY_NEVER_INLINE
|
||||
bool last_byte_error(const exception_id id_, const std::string& detail, const char* context, const char* tail = nullptr)
|
||||
{
|
||||
return report_error(chars_read, get_token_string(), parse_error::create(id_, chars_read, exception_message(detail, context), nullptr));
|
||||
// with a tail, the message reads "<detail><tail>; last byte: 0x<token>"
|
||||
return report_error(chars_read, get_token_string(), parse_error::create(id_, chars_read,
|
||||
exception_message(tail == nullptr ? detail : concat(detail, tail, "; last byte: 0x", get_token_string()), context), nullptr));
|
||||
}
|
||||
|
||||
/*!
|
||||
@@ -5237,7 +5292,8 @@ class binary_reader
|
||||
@param[in] context further context information
|
||||
@return false, so that the caller stops reading (see report_error)
|
||||
*/
|
||||
bool unexpected_byte(const char* detail, const std::string& context)
|
||||
JSON_HEDLEY_NEVER_INLINE
|
||||
bool unexpected_byte(const char* detail, const char* context)
|
||||
{
|
||||
return last_byte_error(exception_id::unexpected_byte, concat(detail, ": 0x", get_token_string()), context);
|
||||
}
|
||||
@@ -5248,6 +5304,7 @@ class binary_reader
|
||||
@param[in] context further context information
|
||||
@return false, so that the caller stops reading (see report_error)
|
||||
*/
|
||||
JSON_HEDLEY_NEVER_INLINE
|
||||
bool length_type_error(const char* position, const char* context)
|
||||
{
|
||||
const char* types = input_format == input_format_t::bjdata ? "U, i, u, I, m, l, M, L" : "U, i, I, l, L";
|
||||
|
||||
@@ -8,12 +8,12 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <algorithm> // min
|
||||
#include <algorithm> // count, find, min
|
||||
#include <array> // array
|
||||
#include <cstddef> // size_t
|
||||
#include <cstdint> // uint32_t
|
||||
#include <cstring> // strlen
|
||||
#include <iterator> // begin, end, iterator_traits, random_access_iterator_tag, distance, next
|
||||
#include <iterator> // begin, end, iterator_traits, random_access_iterator_tag, distance, next, reverse_iterator
|
||||
#include <streambuf> // streambuf
|
||||
#include <string> // string, char_traits
|
||||
#include <type_traits> // enable_if, is_base_of, is_pointer, is_integral, remove_pointer
|
||||
@@ -297,6 +297,7 @@ class iterator_input_adapter
|
||||
return result;
|
||||
}
|
||||
|
||||
++eof_reads;
|
||||
return char_traits<char_type>::eof();
|
||||
}
|
||||
|
||||
@@ -306,6 +307,26 @@ class iterator_input_adapter
|
||||
return static_cast<std::size_t>(std::distance(begin, current));
|
||||
}
|
||||
|
||||
// number of get_character() calls so far, including those that returned
|
||||
// EOF; the lexer derives its position from it (see lexer::lazy_position)
|
||||
std::size_t get_read_count() const
|
||||
{
|
||||
return get_consumed_count() + eof_reads;
|
||||
}
|
||||
|
||||
// number of newlines in the consumed range [0, last_index); @a line_start
|
||||
// is set to the index after the last of them (0 if there is none); only
|
||||
// valid when supports_seek
|
||||
std::size_t count_consumed_lines(std::size_t last_index, std::size_t& line_start) const
|
||||
{
|
||||
const auto to = std::next(begin, static_cast<typename std::iterator_traits<IteratorType>::difference_type>(last_index));
|
||||
const auto newline = char_traits<char_type>::to_char_type('\n');
|
||||
using reverse_it = std::reverse_iterator<IteratorType>;
|
||||
const auto last_newline = std::find(reverse_it(to), reverse_it(begin), newline);
|
||||
line_start = static_cast<std::size_t>(std::distance(begin, last_newline.base()));
|
||||
return static_cast<std::size_t>(std::count(begin, last_newline.base(), newline));
|
||||
}
|
||||
|
||||
// append the already-consumed characters in the half-open range
|
||||
// [first_index, last_index) to @a out; only valid when supports_seek
|
||||
template<typename ContainerType>
|
||||
@@ -422,6 +443,8 @@ class iterator_input_adapter
|
||||
IteratorType begin;
|
||||
IteratorType current;
|
||||
SentinelType end;
|
||||
/// number of get_character() calls that returned EOF
|
||||
std::size_t eof_reads = 0;
|
||||
|
||||
template<typename BaseInputAdapter, size_t T>
|
||||
friend struct wide_string_input_helper;
|
||||
|
||||
@@ -189,12 +189,19 @@ Shared by json_sax_dom_parser and json_sax_dom_callback_parser.
|
||||
@param[in] ref the object or array that was just created
|
||||
@return whether parsing should continue (false after reporting out_of_range.408)
|
||||
*/
|
||||
template<typename SAX, typename BasicJsonType>
|
||||
JSON_HEDLEY_NEVER_INLINE
|
||||
bool container_too_large_error(SAX& sax, std::size_t len, const char* kind, BasicJsonType* ref)
|
||||
{
|
||||
return sax.parse_error(0, "", out_of_range::create(exception_id::container_too_large, concat("excessive ", kind, " size: ", std::to_string(len)), ref));
|
||||
}
|
||||
|
||||
template<typename SAX, typename BasicJsonType>
|
||||
bool check_container_size(SAX& sax, std::size_t len, const char* kind, BasicJsonType* ref)
|
||||
{
|
||||
if (JSON_HEDLEY_UNLIKELY(len != detail::unknown_size() && len > ref->max_size()))
|
||||
{
|
||||
return sax.parse_error(0, "", out_of_range::create(exception_id::container_too_large, concat("excessive ", kind, " size: ", std::to_string(len)), ref));
|
||||
return container_too_large_error(sax, len, kind, ref);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
@@ -219,7 +226,7 @@ struct diagnostic_positions
|
||||
{
|
||||
// Lexer has read the first character of the container, so
|
||||
// subtract 1 from the position to get the correct start position.
|
||||
v.start_position = lexer->get_position() - 1;
|
||||
v.start_position = lexer->get_chars_read_total() - 1;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -233,7 +240,7 @@ struct diagnostic_positions
|
||||
if (lexer)
|
||||
{
|
||||
// Lexer's position is past the closing brace or bracket, so set that as the end position.
|
||||
v.end_position = lexer->get_position();
|
||||
v.end_position = lexer->get_chars_read_total();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -249,7 +256,7 @@ struct diagnostic_positions
|
||||
// Lexer has read past the current field value, so set the end position to the current position.
|
||||
// The start position will be set below based on the length of the string representation
|
||||
// of the value.
|
||||
v.end_position = lexer->get_position();
|
||||
v.end_position = lexer->get_chars_read_total();
|
||||
|
||||
switch (v.type())
|
||||
{
|
||||
@@ -679,16 +686,19 @@ class json_sax_dom_callback_parser
|
||||
if (keep && ref_stack.back())
|
||||
{
|
||||
auto& obj = *ref_stack.back()->m_data.m_value.object;
|
||||
const auto it = obj.find(val);
|
||||
if (it != obj.end())
|
||||
// a single lookup: insert the placeholder, or find the existing member
|
||||
const auto inserted = obj.emplace(val, discarded);
|
||||
const auto it = inserted.first;
|
||||
if (!inserted.second)
|
||||
{
|
||||
// this is a duplicate key (legal in JSON); remember its
|
||||
// current value so it can be restored later if the new
|
||||
// value is rejected by the callback, instead of being
|
||||
// erased together with the discarded placeholder
|
||||
duplicate_key_stash.emplace_back(&(it->second), it->second);
|
||||
it->second = discarded;
|
||||
}
|
||||
object_element = &(obj[val] = discarded);
|
||||
object_element = &(it->second);
|
||||
}
|
||||
|
||||
return true;
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
#include <utility> // move
|
||||
#include <vector> // vector
|
||||
|
||||
#include <nlohmann/detail/bit_ops.hpp>
|
||||
#include <nlohmann/detail/input/input_adapters.hpp>
|
||||
#include <nlohmann/detail/input/number_parse.hpp>
|
||||
#include <nlohmann/detail/input/position_t.hpp>
|
||||
@@ -186,6 +187,15 @@ class lexer : public lexer_base<BasicJsonType>
|
||||
static constexpr bool lazy_token_string =
|
||||
input_adapter_supports_seek<InputAdapterType>(is_detected<detect_supports_seek, InputAdapterType> {});
|
||||
|
||||
/// whether the read position is derived from the input adapter on demand
|
||||
/// instead of being counted on every scanned character. Seekable adapters
|
||||
/// know how much input they consumed and can revisit it, so
|
||||
/// chars_read_total is the adapter's read count (minus a pending unget),
|
||||
/// and line and column are recomputed from the consumed input only when
|
||||
/// get_position() is called (errors). Counting per character kept two
|
||||
/// increments and a newline check in the hot path of every get().
|
||||
static constexpr bool lazy_position = lazy_token_string;
|
||||
|
||||
/// whether a simulated unget can be passed on to the input adapter, which
|
||||
/// then leaves the character in the input; see
|
||||
/// input_adapter_supports_lookahead
|
||||
@@ -195,7 +205,9 @@ class lexer : public lexer_base<BasicJsonType>
|
||||
/// whether string scanning may bulk-consume runs of ordinary characters
|
||||
/// directly from a contiguous input buffer (SWAR fast path). This requires
|
||||
/// the token to be reconstructible lazily (lazy_token_string), so bypassing
|
||||
/// the per-character capture in get() cannot lose error diagnostics.
|
||||
/// the per-character capture in get() cannot lose error diagnostics. It
|
||||
/// also implies lazy_position, so the bulk paths need not advance the
|
||||
/// position counters.
|
||||
static constexpr bool bulk_scan =
|
||||
lazy_token_string
|
||||
&& input_adapter_supports_bulk_scan<InputAdapterType>(is_detected<detect_supports_bulk_scan, InputAdapterType> {});
|
||||
@@ -221,6 +233,41 @@ class lexer : public lexer_base<BasicJsonType>
|
||||
// scan functions
|
||||
/////////////////////
|
||||
|
||||
/// contiguous input: try to decode the 4 hex digits following `\\u`
|
||||
/// directly from the input buffer via hex_codepoint(), instead of 4 calls
|
||||
/// to get(). On success, advances the adapter exactly as those 4 get()
|
||||
/// calls would (bulk_scan implies lazy_position, so there are no position
|
||||
/// counters to advance) and leaves @a current holding the last of
|
||||
/// the 4 digits, just as the last such get() would; the codepoint is
|
||||
/// written to @a out. Makes no state change and returns false for a
|
||||
/// pending unget, fewer than 4 remaining bytes, or any of the 4 bytes not
|
||||
/// being a hex digit, so the caller falls back unchanged to the
|
||||
/// per-character loop, which then reports the same diagnostic (stopping
|
||||
/// at the first invalid digit) as before this optimization.
|
||||
bool get_codepoint_bulk(std::true_type /*bulk*/, int& out)
|
||||
{
|
||||
if (next_unget || ia.bulk_remaining() < 4)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
const char_type* const raw = ia.bulk_data();
|
||||
const int codepoint = hex_codepoint(reinterpret_cast<const unsigned char*>(raw));
|
||||
if (codepoint < 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
ia.bulk_skip(4);
|
||||
current = char_traits<char_type>::to_int_type(raw[3]);
|
||||
out = codepoint;
|
||||
return true;
|
||||
}
|
||||
|
||||
/// streaming input: no bulk fast path
|
||||
bool get_codepoint_bulk(std::false_type /*bulk*/, int& /*out*/) const noexcept
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief get codepoint from 4 hex characters following `\\u`
|
||||
|
||||
@@ -240,6 +287,14 @@ class lexer : public lexer_base<BasicJsonType>
|
||||
{
|
||||
// this function only makes sense after reading `\u`
|
||||
JSON_ASSERT(current == 'u');
|
||||
|
||||
// contiguous input: decode all 4 hex digits directly from the buffer
|
||||
int fast_codepoint = 0;
|
||||
if (get_codepoint_bulk(std::integral_constant<bool, bulk_scan> {}, fast_codepoint))
|
||||
{
|
||||
return fast_codepoint;
|
||||
}
|
||||
|
||||
int codepoint = 0;
|
||||
|
||||
const auto factors = { 12u, 8u, 4u, 0u };
|
||||
@@ -331,10 +386,6 @@ class lexer : public lexer_base<BasicJsonType>
|
||||
}
|
||||
token_buffer.append(reinterpret_cast<const typename string_t::value_type*>(data), pos);
|
||||
ia.bulk_skip(pos);
|
||||
// the run contains no newline (all bytes < 0x20 are treated as special),
|
||||
// so only the flat character counters advance
|
||||
position.chars_read_total += pos;
|
||||
position.chars_read_current_line += pos;
|
||||
}
|
||||
|
||||
/// streaming input: no bulk fast path
|
||||
@@ -1058,9 +1109,11 @@ class lexer : public lexer_base<BasicJsonType>
|
||||
token_type::parse_error otherwise
|
||||
|
||||
@note The scanner is independent of the current locale: token_buffer
|
||||
always holds `.`. Only the std::strtod fallback of convert_number()
|
||||
depends on the locale, and it looks up the decimal point right
|
||||
before converting (see detail::convert_float_locale_aware()).
|
||||
always holds `.`. The conversion of float and double does not use
|
||||
the locale either. Only the std::strtold fallback of
|
||||
convert_number() for long double formats other than binary64
|
||||
depends on it, and it looks up the decimal point right before
|
||||
converting (see detail::convert_float_locale_aware()).
|
||||
*/
|
||||
token_type scan_number() // lgtm [cpp/use-of-goto] `goto` is used in this function to implement the number-parsing state machine described above. By design, any finite input will eventually reach the "done" state or return token_type::parse_error. In each intermediate state, 1 byte of the input is appended to the token_buffer vector, and only the already initialized variables token_buffer, number_type, and error_message are manipulated.
|
||||
{
|
||||
@@ -1073,7 +1126,7 @@ class lexer : public lexer_base<BasicJsonType>
|
||||
|
||||
// offset just past the last mantissa byte in token_buffer (i.e. the
|
||||
// index of 'e'/'E', or the whole token when there is no exponent).
|
||||
// convert_number() uses it to count significant digits; npos means
|
||||
// convert_number() uses it to split the token; npos means
|
||||
// "not seen an exponent yet" and is resolved at scan_number_done
|
||||
std::size_t mantissa_end = std::string::npos;
|
||||
|
||||
@@ -1403,8 +1456,8 @@ scan_number_done:
|
||||
@param[in] mantissa_end offset just past the last mantissa byte in
|
||||
token_buffer (the index of 'e'/'E', or
|
||||
token_buffer.size() when there is no exponent);
|
||||
used to skip Clinger's fast path when it cannot
|
||||
possibly succeed - see detail::mantissa_fits_clinger()
|
||||
with decimal_point_position, it locates the parts
|
||||
of a float token without scanning it again
|
||||
*/
|
||||
token_type convert_number(token_type number_type, std::size_t mantissa_end)
|
||||
{
|
||||
@@ -1458,10 +1511,11 @@ scan_number_done:
|
||||
}
|
||||
|
||||
// this code is reached if we parse a floating-point number or if an
|
||||
// integer conversion above overflowed. Prefer std::from_chars
|
||||
// (Eisel-Lemire, locale-independent, correctly rounded) when available;
|
||||
// otherwise the exact Clinger fast path (double only); otherwise the
|
||||
// locale-aware strtof/strtod/strtold.
|
||||
// integer conversion above overflowed. float and double (and long
|
||||
// double where it is binary64) are converted by the library itself,
|
||||
// correctly rounded and independent of the locale; other long double
|
||||
// formats use std::from_chars when available, otherwise the
|
||||
// locale-aware strtold.
|
||||
if (convert_float_fast(num_begin, num_end, decimal_point_position, mantissa_end, value_float))
|
||||
{
|
||||
return token_type::value_float;
|
||||
@@ -1581,8 +1635,6 @@ scan_number_done:
|
||||
if (JSON_HEDLEY_LIKELY(integer_result != token_type::uninitialized))
|
||||
{
|
||||
ia.bulk_skip(len - 1);
|
||||
position.chars_read_total += (len - 1);
|
||||
position.chars_read_current_line += (len - 1);
|
||||
return integer_result;
|
||||
}
|
||||
// The value does not fit an integer, so this token converts as a
|
||||
@@ -1599,8 +1651,6 @@ scan_number_done:
|
||||
decimal_point_position = dot_index;
|
||||
|
||||
ia.bulk_skip(len - 1);
|
||||
position.chars_read_total += (len - 1);
|
||||
position.chars_read_current_line += (len - 1);
|
||||
|
||||
return convert_number(number_type, mantissa_end);
|
||||
}
|
||||
@@ -1651,7 +1701,7 @@ scan_number_done:
|
||||
|
||||
#if JSON_DIAGNOSTIC_POSITIONS
|
||||
// the first character of the token has already been read, hence the -1
|
||||
token_start_position = position.chars_read_total - 1;
|
||||
token_start_position = get_chars_read_total() - 1;
|
||||
#endif
|
||||
|
||||
note_token_start(std::integral_constant<bool, lazy_token_string> {});
|
||||
@@ -1704,6 +1754,15 @@ scan_number_done:
|
||||
/// per-character position counters (line-count-on-'\n' bookkeeping is
|
||||
/// handled afterwards, in track_after_read(), once `current` is known)
|
||||
void advance_position() noexcept
|
||||
{
|
||||
advance_position_impl(std::integral_constant<bool, lazy_position> {});
|
||||
}
|
||||
|
||||
/// seekable adapter: the position is derived from the adapter on demand
|
||||
void advance_position_impl(std::true_type /*lazy*/) const noexcept {}
|
||||
|
||||
/// streaming adapter: count the character
|
||||
void advance_position_impl(std::false_type /*lazy*/) noexcept
|
||||
{
|
||||
++position.chars_read_total;
|
||||
++position.chars_read_current_line;
|
||||
@@ -1717,7 +1776,16 @@ scan_number_done:
|
||||
// seekable adapters reconstruct the token lazily on error (see
|
||||
// get_token_string), so the eager per-character copy is skipped
|
||||
capture_char(std::integral_constant<bool, lazy_token_string> {});
|
||||
track_newline(std::integral_constant<bool, lazy_position> {});
|
||||
return current;
|
||||
}
|
||||
|
||||
/// seekable adapter: lines are counted on demand in get_position()
|
||||
void track_newline(std::true_type /*lazy*/) const noexcept {}
|
||||
|
||||
/// streaming adapter: start a new line after reading a newline
|
||||
void track_newline(std::false_type /*lazy*/) noexcept
|
||||
{
|
||||
if (current == '\n')
|
||||
{
|
||||
++position.lines_read;
|
||||
@@ -1726,8 +1794,6 @@ scan_number_done:
|
||||
chars_read_before_newline = position.chars_read_current_line;
|
||||
position.chars_read_current_line = 0;
|
||||
}
|
||||
|
||||
return current;
|
||||
}
|
||||
|
||||
/*!
|
||||
@@ -1771,12 +1837,22 @@ scan_number_done:
|
||||
We implement unget by setting variable next_unget to true. The input is not
|
||||
changed - we just simulate ungetting by modifying chars_read_total,
|
||||
chars_read_current_line, and token_string. The next call to get() will
|
||||
behave as if the unget character is read again.
|
||||
behave as if the unget character is read again. With lazy_position, the
|
||||
pending next_unget is all get_chars_read_total() needs.
|
||||
*/
|
||||
void unget()
|
||||
{
|
||||
next_unget = true;
|
||||
unget_position(std::integral_constant<bool, lazy_position> {});
|
||||
uncapture_char(std::integral_constant<bool, lazy_token_string> {});
|
||||
}
|
||||
|
||||
/// seekable adapter: get_chars_read_total() accounts for next_unget
|
||||
void unget_position(std::true_type /*lazy*/) const noexcept {}
|
||||
|
||||
/// streaming adapter: step the counters back over the ungotten character
|
||||
void unget_position(std::false_type /*lazy*/) noexcept
|
||||
{
|
||||
--position.chars_read_total;
|
||||
|
||||
// in case we "unget" a newline, we have to also decrement the lines_read
|
||||
@@ -1799,8 +1875,6 @@ scan_number_done:
|
||||
{
|
||||
--position.chars_read_current_line;
|
||||
}
|
||||
|
||||
uncapture_char(std::integral_constant<bool, lazy_token_string> {});
|
||||
}
|
||||
|
||||
/// adapter without lookahead: nothing to do (see release_lookahead)
|
||||
@@ -1871,10 +1945,21 @@ scan_number_done:
|
||||
// diagnostics
|
||||
/////////////////////
|
||||
|
||||
/// return position of last read token
|
||||
constexpr position_t get_position() const noexcept
|
||||
/*!
|
||||
@brief return the number of characters read so far
|
||||
|
||||
Unlike get_position(), this is O(1) for all adapters, so it may be called
|
||||
for every value (JSON_DIAGNOSTIC_POSITIONS).
|
||||
*/
|
||||
std::size_t get_chars_read_total() const noexcept
|
||||
{
|
||||
return position;
|
||||
return get_chars_read_total_impl(std::integral_constant<bool, lazy_position> {});
|
||||
}
|
||||
|
||||
/// return position of last read token
|
||||
position_t get_position() const
|
||||
{
|
||||
return get_position_impl(std::integral_constant<bool, lazy_position> {});
|
||||
}
|
||||
|
||||
/*!
|
||||
@@ -1911,6 +1996,43 @@ scan_number_done:
|
||||
}
|
||||
#endif
|
||||
|
||||
private:
|
||||
/// seekable adapter: every get() either read from the adapter (including
|
||||
/// reads at EOF, which get() counts as characters as well) or replayed an
|
||||
/// ungotten character, so the count is the adapter's read count minus a
|
||||
/// still pending unget
|
||||
std::size_t get_chars_read_total_impl(std::true_type /*lazy*/) const noexcept
|
||||
{
|
||||
return ia.get_read_count() - (next_unget ? 1u : 0u);
|
||||
}
|
||||
|
||||
/// streaming adapter: the counter is maintained by get() and unget()
|
||||
std::size_t get_chars_read_total_impl(std::false_type /*lazy*/) const noexcept
|
||||
{
|
||||
return position.chars_read_total;
|
||||
}
|
||||
|
||||
/// seekable adapter: count the lines in the consumed input; O(input read
|
||||
/// so far), but only called on errors
|
||||
position_t get_position_impl(std::true_type /*lazy*/) const
|
||||
{
|
||||
position_t result;
|
||||
result.chars_read_total = get_chars_read_total();
|
||||
// reads at EOF count as characters but are not part of the input
|
||||
const std::size_t consumed = (std::min)(result.chars_read_total, ia.get_consumed_count());
|
||||
std::size_t line_start = 0;
|
||||
result.lines_read = ia.count_consumed_lines(consumed, line_start);
|
||||
result.chars_read_current_line = result.chars_read_total - line_start;
|
||||
return result;
|
||||
}
|
||||
|
||||
/// streaming adapter: the position is maintained by get() and unget()
|
||||
position_t get_position_impl(std::false_type /*lazy*/) const noexcept
|
||||
{
|
||||
return position;
|
||||
}
|
||||
|
||||
public:
|
||||
/// seekable adapter: rebuild the last read token from the input on demand
|
||||
const std::vector<char_type>& collect_token_chars(std::vector<char_type>& out, std::true_type /*lazy*/) const
|
||||
{
|
||||
@@ -2018,15 +2140,57 @@ scan_number_done:
|
||||
// into one that reloads the pointer from memory every character
|
||||
do
|
||||
{
|
||||
if (current == '\n')
|
||||
{
|
||||
skip_indentation(std::integral_constant<bool, bulk_scan> {});
|
||||
}
|
||||
get_ignoring_pending_unget();
|
||||
}
|
||||
while (current_is_whitespace());
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief contiguous input: skip the space indentation that follows a newline
|
||||
8 bytes at a time
|
||||
|
||||
Pretty-printed JSON indents every line with a run of spaces, which the loop
|
||||
in skip_whitespace() would otherwise read one character at a time. The
|
||||
spaces are skipped 8 bytes at a time; the last 7 bytes of the input and any
|
||||
other whitespace are left to that loop. Skipping bypasses get(), which is
|
||||
fine because bulk_scan implies lazy_position: the position is derived from
|
||||
the adapter, not counted per character.
|
||||
*/
|
||||
void skip_indentation(std::true_type /*bulk*/)
|
||||
{
|
||||
JSON_ASSERT(!next_unget);
|
||||
const std::size_t remaining = ia.bulk_remaining();
|
||||
if (remaining < 8)
|
||||
{
|
||||
return;
|
||||
}
|
||||
const char* const data = reinterpret_cast<const char*>(ia.bulk_data());
|
||||
std::size_t pos = 0;
|
||||
for (; pos + 8 <= remaining; pos += 8)
|
||||
{
|
||||
// little-endian, so the first byte is the lowest
|
||||
const std::uint64_t non_space = read_eight_bytes(data + pos) ^ 0x2020202020202020ull;
|
||||
if (non_space != 0)
|
||||
{
|
||||
// skip the spaces before the first other byte of this word
|
||||
pos += static_cast<std::size_t>(count_trailing_zeros(non_space)) / 8;
|
||||
break;
|
||||
}
|
||||
}
|
||||
ia.bulk_skip(pos);
|
||||
}
|
||||
|
||||
/// streaming input: no bulk fast path
|
||||
void skip_indentation(std::false_type /*bulk*/) const noexcept {}
|
||||
|
||||
token_type scan()
|
||||
{
|
||||
// initially, skip the BOM
|
||||
if (position.chars_read_total == 0 && !skip_bom())
|
||||
if (get_chars_read_total() == 0 && !skip_bom())
|
||||
{
|
||||
error_message = "invalid BOM; must be 0xEF 0xBB 0xBF if given";
|
||||
return token_type::parse_error;
|
||||
@@ -2053,7 +2217,7 @@ scan_number_done:
|
||||
token_type scan_expecting(token_type expected_type)
|
||||
{
|
||||
JSON_ASSERT(expected_type == token_type::name_separator || expected_type == token_type::value_separator);
|
||||
JSON_ASSERT(position.chars_read_total > 0);
|
||||
JSON_ASSERT(get_chars_read_total() > 0);
|
||||
const char_int_type expected_char = static_cast<unsigned char>((expected_type == token_type::name_separator) ? ':' : ',');
|
||||
skip_whitespace();
|
||||
if (JSON_HEDLEY_LIKELY(current == expected_char))
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -451,7 +451,7 @@ class parser
|
||||
}
|
||||
case token_type::end_of_input:
|
||||
{
|
||||
if (JSON_HEDLEY_UNLIKELY(m_lexer.get_position().chars_read_total == 1))
|
||||
if (JSON_HEDLEY_UNLIKELY(m_lexer.get_chars_read_total() == 1))
|
||||
{
|
||||
// there is nothing to recover
|
||||
static_cast<void>(report_error(sax, parse_error::create(exception_id::syntax_error, m_lexer.get_position(),
|
||||
@@ -1216,8 +1216,10 @@ class parser
|
||||
{
|
||||
// MSVC 2015 reports C4100 (unreferenced parameter) when SAX::parse_error is static
|
||||
static_cast<void>(sax);
|
||||
return sax.parse_error(m_lexer.get_position(), m_lexer.get_token_string(),
|
||||
parse_error::create(exception_id::syntax_error, m_lexer.get_position(), message, nullptr));
|
||||
// computing the position is O(input read so far) for seekable input
|
||||
const position_t position = m_lexer.get_position();
|
||||
return sax.parse_error(position, m_lexer.get_token_string(),
|
||||
parse_error::create(exception_id::syntax_error, position, message, nullptr));
|
||||
}
|
||||
|
||||
std::string exception_message(const token_type expected, const std::string& context)
|
||||
|
||||
@@ -22,6 +22,10 @@ namespace detail
|
||||
constexpr std::int64_t pow5_128_smallest_power = -342;
|
||||
constexpr std::int64_t pow5_128_largest_power = 308;
|
||||
|
||||
// every entry of pow5_128() holds two 64-bit halves of 5^q, one per covered power of 5
|
||||
static_assert((pow5_128_largest_power - pow5_128_smallest_power + 1) * 2 == 1302,
|
||||
"pow5_128_smallest_power/pow5_128_largest_power must match the size of the pow5_128() table");
|
||||
|
||||
/*!
|
||||
@brief 128-bit approximations of 5^q for q in [-342, 308]
|
||||
|
||||
|
||||
@@ -8,10 +8,12 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <array> // array
|
||||
#include <cstddef> // size_t
|
||||
#include <cstdint> // uint64_t
|
||||
#include <cstdint> // uint64_t, uint8_t
|
||||
#include <cstring> // memcpy
|
||||
|
||||
#include <nlohmann/detail/bit_ops.hpp>
|
||||
#include <nlohmann/detail/macro_scope.hpp>
|
||||
|
||||
// Optional SIMD backend for bulk UTF-8 validation. This is an opt-in external
|
||||
@@ -69,18 +71,12 @@ inline std::size_t find_string_special(const unsigned char* data, std::size_t n)
|
||||
std::size_t i = 0;
|
||||
for (; i + 8 <= n; i += 8)
|
||||
{
|
||||
std::uint64_t word = 0;
|
||||
std::memcpy(&word, data + i, sizeof(word));
|
||||
if (swar_string_special(word) != 0)
|
||||
const std::uint64_t special = swar_string_special(read_eight_bytes(data + i));
|
||||
if (special != 0)
|
||||
{
|
||||
// a special byte is in this word; locate it (endian-agnostic)
|
||||
for (std::size_t j = 0; j < 8; ++j)
|
||||
{
|
||||
if (is_string_special(data[i + j]))
|
||||
{
|
||||
return i + j;
|
||||
}
|
||||
}
|
||||
// the lowest flagged byte is the first special one: the borrows of
|
||||
// the subtractions can only flag bytes above a true hit
|
||||
return i + (static_cast<std::size_t>(count_trailing_zeros(special)) / 8);
|
||||
}
|
||||
}
|
||||
for (; i < n; ++i)
|
||||
@@ -114,8 +110,7 @@ inline std::size_t find_ascii_copyable_run(const unsigned char* data, std::size_
|
||||
std::size_t i = 0;
|
||||
for (; i + 8 <= n; i += 8)
|
||||
{
|
||||
std::uint64_t v = 0;
|
||||
std::memcpy(&v, data + i, sizeof(v));
|
||||
const std::uint64_t v = read_eight_bytes(data + i);
|
||||
const std::uint64_t q = v ^ 0x2222222222222222ull; // '"' (0x22)
|
||||
const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; // '\\' (0x5C)
|
||||
const std::uint64_t d = v ^ 0x7F7F7F7F7F7F7F7Full; // DEL (0x7F)
|
||||
@@ -126,7 +121,9 @@ inline std::size_t find_ascii_copyable_run(const unsigned char* data, std::size_
|
||||
| (v & high); // >= 0x80
|
||||
if (stop != 0)
|
||||
{
|
||||
break;
|
||||
// the lowest flagged byte is the first one to stop at (see
|
||||
// find_string_special())
|
||||
return i + (static_cast<std::size_t>(count_trailing_zeros(stop)) / 8);
|
||||
}
|
||||
}
|
||||
for (; i < n; ++i)
|
||||
@@ -253,12 +250,18 @@ inline std::size_t scalar_string_bulk_run(const unsigned char* data, std::size_t
|
||||
{
|
||||
break; // end of buffer, or a quote/escape/control byte
|
||||
}
|
||||
const std::size_t seq = validate_one_utf8(data + pos, n - pos);
|
||||
if (seq == 0)
|
||||
// a run of multi-byte sequences (e.g. CJK text) is validated sequence
|
||||
// by sequence without searching for the next special byte in between
|
||||
do
|
||||
{
|
||||
break; // ill-formed or truncated: let the byte path diagnose it
|
||||
const std::size_t seq = validate_one_utf8(data + pos, n - pos);
|
||||
if (seq == 0)
|
||||
{
|
||||
return pos; // ill-formed or truncated: let the byte path diagnose it
|
||||
}
|
||||
pos += seq;
|
||||
}
|
||||
pos += seq;
|
||||
while (pos < n && data[pos] >= 0x80u);
|
||||
}
|
||||
return pos;
|
||||
}
|
||||
@@ -273,8 +276,7 @@ inline std::size_t find_string_delimiter(const unsigned char* data, std::size_t
|
||||
std::size_t i = 0;
|
||||
for (; i + 8 <= n; i += 8)
|
||||
{
|
||||
std::uint64_t v = 0;
|
||||
std::memcpy(&v, data + i, sizeof(v));
|
||||
const std::uint64_t v = read_eight_bytes(data + i);
|
||||
const std::uint64_t q = v ^ 0x2222222222222222ull;
|
||||
const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull;
|
||||
const std::uint64_t hit = ((q - ones) & ~q & high)
|
||||
@@ -282,14 +284,8 @@ inline std::size_t find_string_delimiter(const unsigned char* data, std::size_t
|
||||
| ((v - 0x2020202020202020ull) & ~v & high);
|
||||
if (hit != 0)
|
||||
{
|
||||
for (std::size_t j = 0; j < 8; ++j)
|
||||
{
|
||||
const unsigned char c = data[i + j];
|
||||
if (c == '\"' || c == '\\' || c < 0x20u)
|
||||
{
|
||||
return i + j;
|
||||
}
|
||||
}
|
||||
// the lowest flagged byte is the first delimiter (see find_string_special())
|
||||
return i + (static_cast<std::size_t>(count_trailing_zeros(hit)) / 8);
|
||||
}
|
||||
}
|
||||
for (; i < n; ++i)
|
||||
@@ -320,5 +316,50 @@ inline std::size_t string_bulk_run(const unsigned char* data, std::size_t n) noe
|
||||
return scalar_string_bulk_run(data, n);
|
||||
}
|
||||
|
||||
// Decode the 4 hex digits at [data, data+4) - the digits following a `\u`
|
||||
// escape - into a codepoint 0x0000..0xFFFF via one table lookup per byte
|
||||
// (after yyjson's read_hex_u16), or return -1 if any of the 4 bytes is not a
|
||||
// hex digit ('0'..'9', 'A'..'F', 'a'..'f'). The caller must already have
|
||||
// checked that 4 bytes are available; used by lexer::get_codepoint()'s
|
||||
// contiguous fast path. On -1 it falls back to the byte-at-a-time loop, which
|
||||
// stops at the first invalid digit, so the reported error and position are
|
||||
// unaffected by this fast path.
|
||||
inline int hex_codepoint(const unsigned char* data) noexcept
|
||||
{
|
||||
static const std::array<std::uint8_t, 256> hex_digit_table = // NOLINT(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
{
|
||||
{
|
||||
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 00..0F
|
||||
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 10..1F
|
||||
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 20..2F
|
||||
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 30..3F ('0'..'9')
|
||||
0xFF, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 40..4F ('A'..'F')
|
||||
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 50..5F
|
||||
0xFF, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 60..6F ('a'..'f')
|
||||
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 70..7F
|
||||
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 80..8F
|
||||
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 90..9F
|
||||
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // A0..AF
|
||||
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // B0..BF
|
||||
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // C0..CF
|
||||
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // D0..DF
|
||||
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // E0..EF
|
||||
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF // F0..FF
|
||||
}
|
||||
};
|
||||
|
||||
const std::uint8_t d0 = hex_digit_table[data[0]];
|
||||
const std::uint8_t d1 = hex_digit_table[data[1]];
|
||||
const std::uint8_t d2 = hex_digit_table[data[2]];
|
||||
const std::uint8_t d3 = hex_digit_table[data[3]];
|
||||
// every valid digit is <= 0xF; the combined OR only exceeds it if at
|
||||
// least one of the four bytes was not a hex digit (looked up as 0xFF)
|
||||
if ((d0 | d1 | d2 | d3) > 0x0F)
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
return (d0 << 12) | (d1 << 8) | (d2 << 4) | d3;
|
||||
}
|
||||
|
||||
} // namespace detail
|
||||
NLOHMANN_JSON_NAMESPACE_END
|
||||
Reference in new issue
Block a user