mirror of
https://github.com/nlohmann/json.git
synced 2026-11-07 13:09:21 +00:00
Merge branch claude/perf-contiguous-parser into claude/perf-swar-reduced-form
Signed-off-by: Niels Lohmann <mail@nlohmann.me>
This commit is contained in:
12 files changed
+876
-285
No files matched your search
@@ -4887,6 +4887,14 @@ class binary_reader
|
||||
const char* context,
|
||||
ContainerType& result)
|
||||
{
|
||||
// a string that is in the block of input as a whole is appended from
|
||||
// it, without the zero-filling resize() below
|
||||
constexpr bool from_block = bulk_scan && std::is_same<ContainerType, string_t>::value;
|
||||
if (append_from_block(len, result, std::integral_constant<bool, from_block> {}))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
// upper bound on the number of bytes read (and allocated) per chunk
|
||||
constexpr std::size_t chunk_size = 4096;
|
||||
|
||||
@@ -4923,6 +4931,36 @@ class binary_reader
|
||||
return true;
|
||||
}
|
||||
|
||||
/// append the @a len bytes at the current position to @a result, if the
|
||||
/// input holds them in one block; false (with nothing read) otherwise
|
||||
template<typename NumberType>
|
||||
bool append_from_block(const NumberType len, string_t& result, std::true_type /*block*/)
|
||||
{
|
||||
if (len <= 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
// (len > 0, so the unsigned type holds it; casting a signed char
|
||||
// straight to std::size_t would trip bugprone-signed-char-misuse)
|
||||
const auto ulen = static_cast<typename std::make_unsigned<NumberType>::type>(len);
|
||||
const std::size_t remaining = ia.bulk_remaining();
|
||||
if (static_cast<std::uintmax_t>(ulen) > static_cast<std::uintmax_t>(remaining))
|
||||
{
|
||||
return false; // (the loop reports the premature end)
|
||||
}
|
||||
const auto n = static_cast<std::size_t>(ulen);
|
||||
result.append(reinterpret_cast<const typename string_t::value_type*>(ia.bulk_data()), n);
|
||||
ia.bulk_skip(n);
|
||||
chars_read += n;
|
||||
return true;
|
||||
}
|
||||
|
||||
template<typename NumberType, typename ContainerType>
|
||||
static bool append_from_block(const NumberType /*len*/, ContainerType& /*result*/, std::false_type /*block*/) noexcept
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
/*!
|
||||
@param[in] context further context information (for diagnostics)
|
||||
@return whether the last read character is not EOF
|
||||
|
||||
@@ -37,8 +37,8 @@ namespace detail
|
||||
@brief a parser for JSON text in one contiguous block, for parse(), accept(),
|
||||
and sax_parse()
|
||||
|
||||
The parser and the lexer exchange one token at a time, and the lexer keeps its
|
||||
state (the current character, a pending unget, where the token starts) in
|
||||
The default parser and lexer exchange one token at a time, and the lexer keeps
|
||||
its state (the current character, a pending unget, where the token starts) in
|
||||
members, which every character pays for. This parser reads the block through a
|
||||
local pointer instead, and the grammar, the whitespace and the tokens are one
|
||||
loop, after yyjson's reader: the state stays in registers, and the expected
|
||||
@@ -46,13 +46,13 @@ next token is tested where it is expected instead of being dispatched on.
|
||||
|
||||
It reports what it reads to a SAX handler: the one parse() uses to build the
|
||||
value, the one accept() uses, or the user's handler of sax_parse(). On a byte
|
||||
the library parser would reject, it stops before reporting the token, and
|
||||
returns false: the events up to there are those the library parser reports.
|
||||
The caller then runs the library parser on the same input to report the error
|
||||
the default parser would reject, it stops before reporting the token, and
|
||||
returns false: the events up to there are those the default parser reports.
|
||||
The caller then runs the default parser on the same input to report the error
|
||||
with its exact message and position (report_contiguous_error(),
|
||||
json_sax_error_forwarder), and parse() builds no value from what was read.
|
||||
|
||||
What is accepted, and the values reported, are those of the library parser
|
||||
What is accepted, and the values reported, are those of the default parser
|
||||
with default options: the same number grammar and conversions, the same UTF-8
|
||||
and escape rules, a UTF-8 byte order mark at the start, and, without
|
||||
JSON_STRICT_NUL_HANDLING, a NUL byte as the end of the input.
|
||||
@@ -106,28 +106,28 @@ class contiguous_parser
|
||||
std::vector<std::uint64_t> outer;
|
||||
|
||||
// the current byte, or 0 at the end (no JSON token starts with 0)
|
||||
#define JSON_CONTIGUOUS_CUR() (p != e ? *p : 0)
|
||||
#define JSON_CONTIGUOUS_CUR() current(p, e)
|
||||
// skip whitespace; most tokens have none before them
|
||||
#define JSON_CONTIGUOUS_WS() \
|
||||
do \
|
||||
{ \
|
||||
if (JSON_HEDLEY_UNLIKELY(JSON_CONTIGUOUS_CUR() <= ' ')) \
|
||||
{ \
|
||||
p = skip_whitespace(p, e); \
|
||||
} \
|
||||
} \
|
||||
#define JSON_CONTIGUOUS_WS() \
|
||||
do \
|
||||
{ \
|
||||
if (JSON_HEDLEY_UNLIKELY(JSON_CONTIGUOUS_CUR() <= ' ')) \
|
||||
{ \
|
||||
p = skip_whitespace(p, e); \
|
||||
} \
|
||||
} \
|
||||
while (false)
|
||||
#define JSON_CONTIGUOUS_OPEN(is_object) \
|
||||
do \
|
||||
{ \
|
||||
#define JSON_CONTIGUOUS_OPEN(is_object) \
|
||||
do \
|
||||
{ \
|
||||
if (JSON_HEDLEY_UNLIKELY(depth != 0 && (depth % 64) == 0)) /* NOLINT(readability/braces) */ \
|
||||
{ \
|
||||
outer.push_back(kinds); \
|
||||
kinds = 0; \
|
||||
} \
|
||||
kinds = (kinds << 1u) | static_cast<std::uint64_t>(is_object); \
|
||||
++depth; \
|
||||
} \
|
||||
{ \
|
||||
outer.push_back(kinds); \
|
||||
kinds = 0; \
|
||||
} \
|
||||
kinds = (kinds << 1u) | static_cast<std::uint64_t>(is_object); \
|
||||
++depth; \
|
||||
} \
|
||||
while (false)
|
||||
|
||||
JSON_CONTIGUOUS_WS();
|
||||
@@ -160,107 +160,107 @@ class contiguous_parser
|
||||
// a value in an array or an object: expanded once for each, so that
|
||||
// each dispatch has its own branch history (arrays tend to hold values
|
||||
// of one kind) and the code after a scalar knows where it is
|
||||
#define JSON_CONTIGUOUS_VALUE \
|
||||
switch (JSON_CONTIGUOUS_CUR()) \
|
||||
{ \
|
||||
case '"': \
|
||||
if (JSON_HEDLEY_UNLIKELY(!string(p, false))) \
|
||||
{ \
|
||||
return false; \
|
||||
} \
|
||||
break; \
|
||||
case '{': \
|
||||
++p; \
|
||||
if (JSON_HEDLEY_UNLIKELY(!report(sax->start_object(detail::unknown_size())))) \
|
||||
{ \
|
||||
return false; \
|
||||
} \
|
||||
JSON_CONTIGUOUS_OPEN(1u); \
|
||||
goto object_first; \
|
||||
case '[': \
|
||||
++p; \
|
||||
if (JSON_HEDLEY_UNLIKELY(!report(sax->start_array(detail::unknown_size())))) \
|
||||
{ \
|
||||
return false; \
|
||||
} \
|
||||
JSON_CONTIGUOUS_OPEN(0u); \
|
||||
goto array_first; \
|
||||
case '-': \
|
||||
case '0': \
|
||||
case '1': \
|
||||
case '2': \
|
||||
case '3': \
|
||||
case '4': \
|
||||
case '5': \
|
||||
case '6': \
|
||||
case '7': \
|
||||
case '8': \
|
||||
case '9': \
|
||||
if (JSON_HEDLEY_UNLIKELY(!number(p))) \
|
||||
{ \
|
||||
return false; \
|
||||
} \
|
||||
break; \
|
||||
case 't': \
|
||||
if (JSON_HEDLEY_UNLIKELY(!literal(p, "true") || !report(sax->boolean(true)))) \
|
||||
{ \
|
||||
return false; \
|
||||
} \
|
||||
break; \
|
||||
case 'f': \
|
||||
if (JSON_HEDLEY_UNLIKELY(!literal(p, "false") || !report(sax->boolean(false)))) \
|
||||
{ \
|
||||
return false; \
|
||||
} \
|
||||
break; \
|
||||
case 'n': \
|
||||
if (JSON_HEDLEY_UNLIKELY(!literal(p, "null") || !report(sax->null()))) \
|
||||
{ \
|
||||
return false; \
|
||||
} \
|
||||
break; \
|
||||
default: \
|
||||
return false; \
|
||||
#define JSON_CONTIGUOUS_VALUE \
|
||||
switch (JSON_CONTIGUOUS_CUR()) \
|
||||
{ \
|
||||
case '"': \
|
||||
if (JSON_HEDLEY_UNLIKELY(!string(p, false))) \
|
||||
{ \
|
||||
return false; \
|
||||
} \
|
||||
break; \
|
||||
case '{': \
|
||||
++p; \
|
||||
if (JSON_HEDLEY_UNLIKELY(!report(sax->start_object(detail::unknown_size())))) \
|
||||
{ \
|
||||
return false; \
|
||||
} \
|
||||
JSON_CONTIGUOUS_OPEN(1u); \
|
||||
goto object_first; \
|
||||
case '[': \
|
||||
++p; \
|
||||
if (JSON_HEDLEY_UNLIKELY(!report(sax->start_array(detail::unknown_size())))) \
|
||||
{ \
|
||||
return false; \
|
||||
} \
|
||||
JSON_CONTIGUOUS_OPEN(0u); \
|
||||
goto array_first; \
|
||||
case '-': \
|
||||
case '0': \
|
||||
case '1': \
|
||||
case '2': \
|
||||
case '3': \
|
||||
case '4': \
|
||||
case '5': \
|
||||
case '6': \
|
||||
case '7': \
|
||||
case '8': \
|
||||
case '9': \
|
||||
if (JSON_HEDLEY_UNLIKELY(!number(p))) \
|
||||
{ \
|
||||
return false; \
|
||||
} \
|
||||
break; \
|
||||
case 't': \
|
||||
if (JSON_HEDLEY_UNLIKELY(!literal(p, "true") || !report(sax->boolean(true)))) \
|
||||
{ \
|
||||
return false; \
|
||||
} \
|
||||
break; \
|
||||
case 'f': \
|
||||
if (JSON_HEDLEY_UNLIKELY(!literal(p, "false") || !report(sax->boolean(false)))) \
|
||||
{ \
|
||||
return false; \
|
||||
} \
|
||||
break; \
|
||||
case 'n': \
|
||||
if (JSON_HEDLEY_UNLIKELY(!literal(p, "null") || !report(sax->null()))) \
|
||||
{ \
|
||||
return false; \
|
||||
} \
|
||||
break; \
|
||||
default: \
|
||||
return false; \
|
||||
}
|
||||
|
||||
// what may follow a value in an array: a comma and the next value, or the end
|
||||
#define JSON_CONTIGUOUS_AFTER_ELEMENT \
|
||||
JSON_CONTIGUOUS_WS(); \
|
||||
#define JSON_CONTIGUOUS_AFTER_ELEMENT \
|
||||
JSON_CONTIGUOUS_WS(); \
|
||||
if (JSON_HEDLEY_LIKELY(JSON_CONTIGUOUS_CUR() == ',')) /* NOLINT(readability/braces) */ \
|
||||
{ \
|
||||
++p; \
|
||||
JSON_CONTIGUOUS_WS(); \
|
||||
goto array_value; \
|
||||
} \
|
||||
{ \
|
||||
++p; \
|
||||
JSON_CONTIGUOUS_WS(); \
|
||||
goto array_value; \
|
||||
} \
|
||||
if (JSON_HEDLEY_LIKELY(JSON_CONTIGUOUS_CUR() == ']')) /* NOLINT(readability/braces) */ \
|
||||
{ \
|
||||
++p; \
|
||||
if (JSON_HEDLEY_UNLIKELY(!report(sax->end_array()))) \
|
||||
{ \
|
||||
return false; \
|
||||
} \
|
||||
goto close; \
|
||||
} \
|
||||
{ \
|
||||
++p; \
|
||||
if (JSON_HEDLEY_UNLIKELY(!report(sax->end_array()))) \
|
||||
{ \
|
||||
return false; \
|
||||
} \
|
||||
goto close; \
|
||||
} \
|
||||
return false;
|
||||
|
||||
// what may follow a value in an object: a comma and the next member, or the end
|
||||
#define JSON_CONTIGUOUS_AFTER_MEMBER \
|
||||
JSON_CONTIGUOUS_WS(); \
|
||||
#define JSON_CONTIGUOUS_AFTER_MEMBER \
|
||||
JSON_CONTIGUOUS_WS(); \
|
||||
if (JSON_HEDLEY_LIKELY(JSON_CONTIGUOUS_CUR() == ',')) /* NOLINT(readability/braces) */ \
|
||||
{ \
|
||||
++p; \
|
||||
JSON_CONTIGUOUS_WS(); \
|
||||
goto object_key; \
|
||||
} \
|
||||
{ \
|
||||
++p; \
|
||||
JSON_CONTIGUOUS_WS(); \
|
||||
goto object_key; \
|
||||
} \
|
||||
if (JSON_HEDLEY_LIKELY(JSON_CONTIGUOUS_CUR() == '}')) /* NOLINT(readability/braces) */ \
|
||||
{ \
|
||||
++p; \
|
||||
if (JSON_HEDLEY_UNLIKELY(!report(sax->end_object()))) \
|
||||
{ \
|
||||
return false; \
|
||||
} \
|
||||
goto close; \
|
||||
} \
|
||||
{ \
|
||||
++p; \
|
||||
if (JSON_HEDLEY_UNLIKELY(!report(sax->end_object()))) \
|
||||
{ \
|
||||
return false; \
|
||||
} \
|
||||
goto close; \
|
||||
} \
|
||||
return false;
|
||||
|
||||
array_first:
|
||||
@@ -387,6 +387,12 @@ done:
|
||||
return KeepValues;
|
||||
}
|
||||
|
||||
/// the byte at @a p, or 0 at the end @a e (no JSON token starts with 0)
|
||||
static JSON_HEDLEY_ALWAYS_INLINE uchar current(const uchar* p, const uchar* e) noexcept
|
||||
{
|
||||
return p != e ? *p : 0;
|
||||
}
|
||||
|
||||
/// the first byte at or after p that is not whitespace, or e
|
||||
static JSON_HEDLEY_NEVER_INLINE const uchar* skip_whitespace(const uchar* p, const uchar* e) noexcept
|
||||
{
|
||||
@@ -436,7 +442,7 @@ done:
|
||||
/// a JSON text that is a single scalar
|
||||
bool scalar(const uchar*& p)
|
||||
{
|
||||
switch (p != end ? *p : 0)
|
||||
switch (current(p, end))
|
||||
{
|
||||
case '"':
|
||||
return string(p, false);
|
||||
@@ -624,7 +630,7 @@ done:
|
||||
contiguous path (lexer::scan_number_bulk_contiguous())
|
||||
|
||||
Integers that do not fit their type become floats; floats that overflow
|
||||
are an error, which the library parser reports.
|
||||
are an error, which the default parser reports.
|
||||
*/
|
||||
JSON_HEDLEY_ALWAYS_INLINE bool number(const uchar*& pos)
|
||||
{
|
||||
@@ -773,7 +779,7 @@ done:
|
||||
}
|
||||
if (JSON_HEDLEY_UNLIKELY(!std::isfinite(value)))
|
||||
{
|
||||
return false; // the library parser reports the overflow
|
||||
return false; // the default parser reports the overflow
|
||||
}
|
||||
return report(sax->number_float(value, number_text(data, data + i)));
|
||||
}
|
||||
@@ -811,7 +817,7 @@ struct can_parse_contiguous
|
||||
@brief report the input behind @a adapter, one block of memory, to @a sax
|
||||
|
||||
@return whether the input is valid and was reported completely; if not,
|
||||
nothing was consumed from @a adapter, and the caller has the library
|
||||
nothing was consumed from @a adapter, and the caller has the default
|
||||
parser report the error
|
||||
*/
|
||||
template<typename BasicJsonType, bool KeepValues, typename SAX, typename InputAdapterType>
|
||||
@@ -820,7 +826,7 @@ bool parse_contiguous(InputAdapterType& adapter, SAX& sax, std::true_type /*cont
|
||||
const std::size_t size = adapter.bulk_remaining();
|
||||
if (size == 0)
|
||||
{
|
||||
return false; // the library parser reports the empty input
|
||||
return false; // the default parser reports the empty input
|
||||
}
|
||||
const auto* const first = reinterpret_cast<const unsigned char*>(adapter.bulk_data());
|
||||
contiguous_parser<BasicJsonType, SAX, KeepValues> p(first, first + size, &sax);
|
||||
@@ -831,7 +837,7 @@ bool parse_contiguous(InputAdapterType& adapter, SAX& sax, std::true_type /*cont
|
||||
@brief a SAX handler that only reports a parse error, as the handler of parse()
|
||||
does: it throws the exception, unless exceptions are not allowed
|
||||
|
||||
After contiguous_parser has rejected an input, the library parser is run again
|
||||
After contiguous_parser has rejected an input, the default parser is run again
|
||||
with this handler to report the error: the error, its message and its position
|
||||
are those of parse(), without building the values read up to it.
|
||||
*/
|
||||
@@ -868,7 +874,7 @@ struct sax_ignores_number_text<json_sax_error_reporter<BasicJsonType>> : std::tr
|
||||
rejected, as parse() reports it: throw its exception, or, without
|
||||
exceptions, return true
|
||||
|
||||
@return false if the library parser accepts the input after all (which would
|
||||
@return false if the default parser accepts the input after all (which would
|
||||
be a disagreement of the two parsers): the caller then parses it
|
||||
*/
|
||||
template<typename BasicJsonType, typename InputAdapterType>
|
||||
@@ -922,7 +928,7 @@ bool parse_contiguous(InputAdapterType& /*adapter*/, BasicJsonType& /*result*/,
|
||||
|
||||
@return whether @a accepted is the result of accept(); false if the input is
|
||||
not one block of memory, in which case the caller checks it.
|
||||
contiguous_parser accepts exactly what the library parser accepts.
|
||||
contiguous_parser accepts exactly what the default parser accepts.
|
||||
*/
|
||||
template<typename BasicJsonType, typename InputAdapterType>
|
||||
bool accept_contiguous(InputAdapterType& adapter, bool& accepted, std::true_type /*contiguous*/)
|
||||
@@ -943,9 +949,9 @@ bool accept_contiguous(InputAdapterType& /*adapter*/, bool& /*accepted*/, std::f
|
||||
handler of the user
|
||||
|
||||
After contiguous_parser has delivered the events of the input to the user's
|
||||
handler and rejected it, the library parser is run again with this handler:
|
||||
handler and rejected it, the default parser is run again with this handler:
|
||||
the user's handler gets the error, its message and its position from the
|
||||
library parser, as before, but no event twice.
|
||||
default parser, as before, but no event twice.
|
||||
*/
|
||||
template<typename BasicJsonType, typename SAX>
|
||||
class json_sax_error_forwarder : public json_sax_acceptor<BasicJsonType>
|
||||
@@ -962,7 +968,7 @@ class json_sax_error_forwarder : public json_sax_acceptor<BasicJsonType>
|
||||
return user->parse_error(position, last_token, ex);
|
||||
}
|
||||
|
||||
/// whether the library parser reported an error
|
||||
/// whether the default parser reported an error
|
||||
bool reported() const noexcept
|
||||
{
|
||||
return forwarded;
|
||||
@@ -981,8 +987,8 @@ struct sax_ignores_number_text<json_sax_error_forwarder<BasicJsonType, SAX>> : s
|
||||
a user's handler @a sax
|
||||
|
||||
contiguous_parser reports to @a sax what it reads, and stops at the token that
|
||||
the library parser would reject, before reporting it: the events up to an error
|
||||
are the same. The error itself is then reported by the library parser (see
|
||||
the default parser would reject, before reporting it: the events up to an error
|
||||
are the same. The error itself is then reported by the default parser (see
|
||||
json_sax_error_forwarder).
|
||||
|
||||
@return whether @a result is the result of sax_parse(); false if the input is
|
||||
@@ -1003,14 +1009,14 @@ bool sax_parse_contiguous(InputAdapterType& adapter, SAX* sax, bool& result, std
|
||||
}
|
||||
if (p.aborted())
|
||||
{
|
||||
// the handler returned false, as with the library parser
|
||||
// the handler returned false, as with the default parser
|
||||
result = false;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
json_sax_error_forwarder<BasicJsonType, SAX> forwarder(sax);
|
||||
result = parser<BasicJsonType, InputAdapterType>(InputAdapterType(adapter), nullptr, true, false, false).sax_parse(&forwarder, true);
|
||||
// the library parser must reject the input as well (the events after the
|
||||
// the default parser must reject the input as well (the events after the
|
||||
// byte contiguous_parser stopped at cannot be delivered any more)
|
||||
JSON_ASSERT(forwarder.reported());
|
||||
return true;
|
||||
|
||||
@@ -58,6 +58,19 @@
|
||||
#define JSON_UNLOCKED_FILE_IO 0
|
||||
#endif
|
||||
#undef JSON_UNLOCKED_FILE_IO_MSAN
|
||||
|
||||
// Whether file_input_adapter scans the buffer of the FILE directly, as
|
||||
// getc_unlocked() reads it: the fields it uses are those that the C
|
||||
// library's own getc_unlocked() macro uses (glibc: _IO_read_ptr and
|
||||
// _IO_read_end; Apple: _p and _r). It needs the file locked for the whole
|
||||
// parse, as JSON_UNLOCKED_FILE_IO == 1 has it (so JSON_DISABLE_UNLOCKED_FILE_IO
|
||||
// and MemorySanitizer builds turn it off).
|
||||
// (uClibc defines __GLIBC__ as well, but its FILE has other fields)
|
||||
#if JSON_UNLOCKED_FILE_IO == 1 && ((defined(__GLIBC__) && !defined(__UCLIBC__)) || defined(__APPLE__))
|
||||
#define JSON_FILE_WINDOW 1
|
||||
#else
|
||||
#define JSON_FILE_WINDOW 0
|
||||
#endif
|
||||
#endif // JSON_NO_IO
|
||||
|
||||
NLOHMANN_JSON_NAMESPACE_BEGIN
|
||||
@@ -179,8 +192,12 @@ class token_carry_buffer
|
||||
char* m_capacity_end = nullptr;
|
||||
};
|
||||
/*!
|
||||
Input adapter for stdio file access. This adapter read only 1 byte and do not use any
|
||||
buffer. This adapter is a very low level adapter.
|
||||
Input adapter for stdio file access. The file is locked for the lifetime of the
|
||||
adapter and read with the unlocked stdio functions (see
|
||||
JSON_DISABLE_UNLOCKED_FILE_IO). With glibc and the Apple C library, the
|
||||
characters in the buffer of the FILE are also scanned in bulk, as the get area
|
||||
of a stream is (see input_stream_adapter): exactly the characters getc_unlocked()
|
||||
would return are consumed, so the file is left where it was left before.
|
||||
*/
|
||||
class file_input_adapter
|
||||
{
|
||||
@@ -199,6 +216,12 @@ class file_input_adapter
|
||||
file_input_adapter(const file_input_adapter&) = delete;
|
||||
file_input_adapter(file_input_adapter&& rhs) noexcept
|
||||
: m_file(rhs.m_file)
|
||||
#if JSON_FILE_WINDOW
|
||||
, token_carry(std::move(rhs.token_carry))
|
||||
, token_segment(rhs.token_segment)
|
||||
, current_in_buffer(rhs.current_in_buffer)
|
||||
, tracking_token(rhs.tracking_token)
|
||||
#endif
|
||||
{
|
||||
// the lock is owned by the thread, not the adapter: hand it over
|
||||
rhs.m_file = nullptr;
|
||||
@@ -210,6 +233,20 @@ class file_input_adapter
|
||||
unlock();
|
||||
}
|
||||
|
||||
#if JSON_FILE_WINDOW
|
||||
std::char_traits<char>::int_type get_character()
|
||||
{
|
||||
// what getc_unlocked() does, while the buffer has characters
|
||||
const char* const next = buffer_ptr();
|
||||
if (JSON_HEDLEY_LIKELY(next != buffer_end()))
|
||||
{
|
||||
current_in_buffer = true;
|
||||
buffer_bump(1);
|
||||
return std::char_traits<char>::to_int_type(*next);
|
||||
}
|
||||
return get_character_slow();
|
||||
}
|
||||
#else
|
||||
std::char_traits<char>::int_type get_character() noexcept
|
||||
{
|
||||
// the file is locked for the lifetime of the adapter, so the
|
||||
@@ -222,6 +259,7 @@ class file_input_adapter
|
||||
return std::fgetc(m_file);
|
||||
#endif
|
||||
}
|
||||
#endif
|
||||
|
||||
// returns the number of characters successfully read
|
||||
template<class T>
|
||||
@@ -230,7 +268,134 @@ class file_input_adapter
|
||||
return fread(dest, 1, sizeof(T) * count, m_file);
|
||||
}
|
||||
|
||||
#if JSON_FILE_WINDOW
|
||||
// The lexer may scan the buffer of the FILE directly, as the window of
|
||||
// input_stream_adapter: the buffer ends wherever the C library's buffer
|
||||
// ends, and is refilled by the next get_character().
|
||||
static constexpr bool supports_bulk_scan = true;
|
||||
static constexpr bool bulk_window = true;
|
||||
|
||||
// Pointer to the next unread character; only valid when bulk_remaining() > 0.
|
||||
const char* bulk_data() const
|
||||
{
|
||||
return buffer_ptr();
|
||||
}
|
||||
|
||||
// Number of unread characters in the buffer.
|
||||
std::size_t bulk_remaining() const
|
||||
{
|
||||
if (!current_in_buffer)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
return static_cast<std::size_t>(buffer_end() - buffer_ptr());
|
||||
}
|
||||
|
||||
// Consume @a n characters previously inspected via bulk_data().
|
||||
void bulk_skip(std::size_t n)
|
||||
{
|
||||
JSON_ASSERT(n <= bulk_remaining());
|
||||
buffer_bump(n);
|
||||
}
|
||||
|
||||
// The adapter keeps the current token for the lexer's error messages, as
|
||||
// input_stream_adapter does.
|
||||
static constexpr bool supports_token_tracking = true;
|
||||
|
||||
void begin_token() noexcept
|
||||
{
|
||||
tracking_token = true;
|
||||
token_carry.clear();
|
||||
token_segment = buffer_ptr();
|
||||
}
|
||||
|
||||
void mark_token_start(std::char_traits<char>::int_type c)
|
||||
{
|
||||
tracking_token = true;
|
||||
token_carry.clear();
|
||||
if (current_in_buffer)
|
||||
{
|
||||
token_segment = buffer_ptr() - 1;
|
||||
}
|
||||
else
|
||||
{
|
||||
token_segment = nullptr;
|
||||
token_carry.push_back(std::char_traits<char>::to_char_type(c));
|
||||
}
|
||||
}
|
||||
|
||||
void copy_token(std::vector<char>& out, bool drop_last) const
|
||||
{
|
||||
token_carry.copy_to(out);
|
||||
if (token_segment != nullptr)
|
||||
{
|
||||
out.insert(out.end(), token_segment, buffer_ptr());
|
||||
}
|
||||
if (drop_last && !out.empty())
|
||||
{
|
||||
out.pop_back();
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
private:
|
||||
#if JSON_FILE_WINDOW
|
||||
// the buffer of the FILE: [buffer_ptr(), buffer_end()) are the characters
|
||||
// getc_unlocked() would return next
|
||||
#if defined(__GLIBC__)
|
||||
const char* buffer_ptr() const noexcept
|
||||
{
|
||||
return m_file->_IO_read_ptr;
|
||||
}
|
||||
const char* buffer_end() const noexcept
|
||||
{
|
||||
return m_file->_IO_read_end;
|
||||
}
|
||||
void buffer_bump(std::size_t n) noexcept
|
||||
{
|
||||
m_file->_IO_read_ptr += n;
|
||||
}
|
||||
#else
|
||||
const char* buffer_ptr() const noexcept
|
||||
{
|
||||
return reinterpret_cast<const char*>(m_file->_p);
|
||||
}
|
||||
const char* buffer_end() const noexcept
|
||||
{
|
||||
return m_file->_r > 0 ? reinterpret_cast<const char*>(m_file->_p) + m_file->_r : reinterpret_cast<const char*>(m_file->_p);
|
||||
}
|
||||
void buffer_bump(std::size_t n) noexcept
|
||||
{
|
||||
m_file->_p += n;
|
||||
m_file->_r -= static_cast<int>(n);
|
||||
}
|
||||
#endif
|
||||
|
||||
// the buffer is exhausted: getc_unlocked() refills it
|
||||
JSON_HEDLEY_NEVER_INLINE std::char_traits<char>::int_type get_character_slow()
|
||||
{
|
||||
if (token_segment != nullptr)
|
||||
{
|
||||
token_carry.append(token_segment, buffer_end());
|
||||
token_segment = nullptr;
|
||||
}
|
||||
const auto res = getc_unlocked(m_file);
|
||||
if (res == EOF)
|
||||
{
|
||||
current_in_buffer = false;
|
||||
return std::char_traits<char>::eof();
|
||||
}
|
||||
// the character was taken from the refilled buffer, at buffer_ptr() - 1
|
||||
current_in_buffer = true;
|
||||
if (tracking_token)
|
||||
{
|
||||
token_segment = buffer_ptr() - 1;
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
// Lock the file once for the whole parse instead of once per character.
|
||||
// The lock is recursive, so fread() and the user's own stdio calls on the
|
||||
// same thread keep working.
|
||||
@@ -259,6 +424,16 @@ class file_input_adapter
|
||||
|
||||
/// the file pointer to read from
|
||||
std::FILE* m_file;
|
||||
#if JSON_FILE_WINDOW
|
||||
/// the part of the current token that was in earlier buffers
|
||||
token_carry_buffer token_carry{}; // NOLINT(readability-redundant-member-init)
|
||||
/// start of the current token's part in the buffer, or nullptr
|
||||
const char* token_segment = nullptr;
|
||||
/// whether the character last returned by get_character() is in the buffer
|
||||
bool current_in_buffer = false;
|
||||
/// whether a token is tracked at all; the binary readers never start one
|
||||
bool tracking_token = false;
|
||||
#endif
|
||||
};
|
||||
|
||||
/*!
|
||||
|
||||
@@ -22,6 +22,7 @@
|
||||
#undef JSON_DISABLE_ENUM_SERIALIZATION
|
||||
#undef JSON_DISABLE_TUPLE_REFERENCE_CONVERSION
|
||||
#undef JSON_UNLOCKED_FILE_IO
|
||||
#undef JSON_FILE_WINDOW
|
||||
#undef JSON_DTOA_SSE2
|
||||
#undef JSON_DTOA_NEON
|
||||
#undef JSON_LITTLE_ENDIAN_TARGET
|
||||
|
||||
@@ -3111,7 +3111,7 @@ public:
|
||||
{
|
||||
basic_json result;
|
||||
// JSON in one block of memory, with the default options, is read by
|
||||
// the faster contiguous_parser; on an error, the library parser reads
|
||||
// the faster contiguous_parser; on an error, the default parser reads
|
||||
// it again to report it (see contiguous_parser)
|
||||
using contiguous = std::integral_constant<bool, detail::can_parse_contiguous<basic_json, typename std::decay<InputAdapterType>::type>::value>;
|
||||
if (cb == nullptr && !ignore_comments && !ignore_trailing_commas
|
||||
@@ -3129,8 +3129,8 @@ public:
|
||||
const bool ignore_comments,
|
||||
const bool ignore_trailing_commas)
|
||||
{
|
||||
// JSON in one block of memory is checked by the faster
|
||||
// contiguous_parser (see parse_impl())
|
||||
// JSON in one block of memory, with the default options, is checked by
|
||||
// the faster contiguous_parser (see parse_impl())
|
||||
using adapter_t = typename std::decay<InputAdapterType>::type;
|
||||
bool accepted = false;
|
||||
if (!ignore_comments && !ignore_trailing_commas
|
||||
@@ -3154,8 +3154,9 @@ public:
|
||||
{
|
||||
if (format == input_format_t::json)
|
||||
{
|
||||
// JSON in one block of memory is read by the faster
|
||||
// contiguous_parser, with the default options (see parse_impl())
|
||||
// JSON in one block of memory, with the default options, is read by
|
||||
// the faster contiguous_parser; on an error, the default parser
|
||||
// reads it again to report it (see parse_impl())
|
||||
bool result = false;
|
||||
if (strict && !ignore_comments && !ignore_trailing_commas
|
||||
&& detail::sax_parse_contiguous<basic_json>(ia, sax, result, std::integral_constant<bool, detail::can_parse_contiguous<basic_json, InputAdapterType>::value> {}))
|
||||
|
||||
Reference in new issue
Block a user