diff --git a/BUILD.bazel b/BUILD.bazel index 8fa0a9a9c..30d04eaea 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -36,6 +36,7 @@ cc_library( "include/nlohmann/detail/input/parser.hpp", "include/nlohmann/detail/input/position_t.hpp", "include/nlohmann/detail/input/pow5_table.hpp", + "include/nlohmann/detail/input/simd_scan.hpp", "include/nlohmann/detail/input/string_scan.hpp", "include/nlohmann/detail/iterators/internal_iterator.hpp", "include/nlohmann/detail/iterators/iter_impl.hpp", diff --git a/docs/mkdocs/docs/api/macros/index.md b/docs/mkdocs/docs/api/macros/index.md index 82e3ac8e3..d72972551 100644 --- a/docs/mkdocs/docs/api/macros/index.md +++ b/docs/mkdocs/docs/api/macros/index.md @@ -38,6 +38,7 @@ header. See also the [macro overview page](../../features/macros.md). - [**JSON_HAS_THREE_WAY_COMPARISON**](json_has_three_way_comparison.md) - control 3-way comparison support - [**JSON_NO_AUTOMATIC_UDLS**](json_no_automatic_udls.md) - do not include the user-defined string literals (UDLs) automatically - [**JSON_NO_IO**](json_no_io.md) - switch off functions relying on certain C++ I/O headers +- [**JSON_NO_SIMD**](json_no_simd.md) - use only portable code instead of vector instructions (SSE2, SSSE3, NEON) - [**JSON_NO_THREAD_LOCAL**](json_no_thread_local.md) - switch off the use of `thread_local` storage - [**JSON_SKIP_UNSUPPORTED_COMPILER_CHECK**](json_skip_unsupported_compiler_check.md) - do not warn about unsupported compilers - [**JSON_USE_GLOBAL_UDLS**](json_use_global_udls.md) - place user-defined string literals (UDLs) into the global namespace diff --git a/docs/mkdocs/docs/api/macros/json_no_simd.md b/docs/mkdocs/docs/api/macros/json_no_simd.md new file mode 100644 index 000000000..088738b7c --- /dev/null +++ b/docs/mkdocs/docs/api/macros/json_no_simd.md @@ -0,0 +1,83 @@ +# JSON_NO_SIMD + +```cpp +#define JSON_NO_SIMD +``` + +When defined, the library uses only portable C++ code where it would otherwise use vector instructions: when scanning +strings during parsing, and when writing floating-point numbers during serialization. + +The macro is an escape hatch, for example for toolchains that cannot compile the vector code, for sanitizers or +analyzers that do not understand it, or to compare the speed of both variants. It does not change any result: values, +exceptions, and error positions are the same either way. Only the speed differs. + +## Default definition + +By default, `#!cpp JSON_NO_SIMD` is not defined, and vector instructions are used where the target has them. + +```cpp +#undef JSON_NO_SIMD +``` + +## Notes + +!!! note "Which instructions are used" + + - **x86-64:** SSE2, which every x86-64 CPU has, finds the end of runs of plain string characters 16 bytes at a time + and writes the digits of floating-point numbers. The UTF-8 check of non-ASCII text needs SSSE3, which is not part + of x86-64: it is compiled for SSSE3 with a function attribute and used when the CPU has SSSE3 (every x86-64 CPU + since about 2011), else the portable check. Code compiled for SSSE3 anyway (e.g., with `-mssse3` or `-march=native`) + skips this check of the CPU. 32-bit x86 uses SSE2 for strings when the compiler targets it. + - **AArch64:** NEON, which every 64-bit Arm CPU has, checks UTF-8 text and long runs of plain string characters 16 + bytes at a time, and writes the digits of floating-point numbers. With MSVC, only the floating-point digits use + NEON. + - **Other targets:** portable code only. + + Strings are only scanned this way for contiguous input and for the characters a `#!cpp std::istream` holds in its + stream buffer (see [`JSON_DISABLE_STREAM_BULK_SCAN`](json_disable_stream_bulk_scan.md)); other inputs are read one + character at a time. + +!!! note "Interaction with `JSON_USE_SIMDUTF`" + + With [`JSON_USE_SIMDUTF`](json_use_simdutf.md), runs of string characters are first validated by simdutf; the code + described here is used for what simdutf does not accept, so that errors are reported at the same position. + `JSON_NO_SIMD` does not affect simdutf. + +!!! warning "Define consistently" + + The macro selects between two definitions of the library's internal scanning and formatting functions. It must + therefore be defined identically for **every** translation unit that includes the library; mixing translation units + that define it with ones that do not is an ODR violation. Prefer setting it as a compile definition on the target + rather than with `#!cpp #define` in individual source files. + +## Examples + +??? example "Example: switching off vector instructions" + + The program parses and serializes the same values with and without the macro: + + ```cpp + #define JSON_NO_SIMD + #include + #include + + using json = nlohmann::json; + + int main() + { + const json j = json::parse(R"({"name": "日本語 text", "pi": 3.141592653589793})"); + std::cout << j.dump() << '\n'; // {"name":"日本語 text","pi":3.141592653589793} + } + ``` + +## See also + +- [**parse**](../basic_json/parse.md) - deserialize from a compatible input +- [**dump**](../basic_json/dump.md) - serialization +- [**JSON_USE_SIMDUTF**](json_use_simdutf.md) - use the simdutf library to accelerate UTF-8 validation +- [**JSON_DISABLE_STREAM_BULK_SCAN**](json_disable_stream_bulk_scan.md) - read a `std::istream` one character at a time +- [Parsing: input sources and performance](../../features/parsing/index.md#input-sources-and-performance) + +## Version history + +- Added in version 3.13.0. diff --git a/docs/mkdocs/docs/api/macros/json_use_simdutf.md b/docs/mkdocs/docs/api/macros/json_use_simdutf.md index 6e3d0406e..5502a77f9 100644 --- a/docs/mkdocs/docs/api/macros/json_use_simdutf.md +++ b/docs/mkdocs/docs/api/macros/json_use_simdutf.md @@ -6,8 +6,9 @@ When defined, the parser validates the UTF-8 content of JSON strings that come from a **contiguous byte input** (`std::string`, `std::vector`/``, string literals, `const char*` ranges, …) using the -[simdutf](https://github.com/simdutf/simdutf) library instead of the built-in scalar validator. On text with many -non-ASCII characters (e.g. CJK or emoji) this can validate several times faster. +[simdutf](https://github.com/simdutf/simdutf) library instead of the built-in validator (which uses SSSE3 or NEON where +available, see [`JSON_NO_SIMD`](json_no_simd.md)). On text with many non-ASCII characters (e.g. CJK or emoji) this can +validate faster, depending on the CPU. This is an **opt-in external dependency**. The library itself remains header-only and its behavior is unchanged: the same input is accepted or rejected either way, and every parse error is reported at the same position with the same diff --git a/docs/mkdocs/docs/features/macros.md b/docs/mkdocs/docs/features/macros.md index 968f92f31..620cbe0da 100644 --- a/docs/mkdocs/docs/features/macros.md +++ b/docs/mkdocs/docs/features/macros.md @@ -174,6 +174,13 @@ security reasons (e.g., Intel Software Guard Extensions (SGX)). See [full documentation of `JSON_NO_IO`](../api/macros/json_no_io.md). +## `JSON_NO_SIMD` + +When defined, the library uses only portable code instead of vector instructions (SSE2, SSSE3, NEON) for scanning +strings and writing floating-point numbers. Results are the same either way. It is not defined by default. + +See [full documentation of `JSON_NO_SIMD`](../api/macros/json_no_simd.md). + ## `JSON_NO_THREAD_LOCAL` When defined, the library does not use `#!cpp thread_local` storage. Copying a value and comparing two values then diff --git a/docs/mkdocs/docs/features/parsing/index.md b/docs/mkdocs/docs/features/parsing/index.md index f823786be..07fb6500d 100644 --- a/docs/mkdocs/docs/features/parsing/index.md +++ b/docs/mkdocs/docs/features/parsing/index.md @@ -79,6 +79,10 @@ follow the usual `std::streambuf` contract, and [`JSON_DISABLE_UNLOCKED_FILE_IO`](../../api/macros/json_disable_unlocked_file_io.md) for programs in which another thread must use the same `#!cpp FILE` object while it is parsed. +Where the target has them, strings are scanned with vector instructions: SSE2 on x86-64 (and SSSE3 for UTF-8 text, when +the CPU has it) and NEON on AArch64. [`JSON_NO_SIMD`](../../api/macros/json_no_simd.md) selects portable code instead, +again without changing any result. + ## SAX vs. DOM parsing The library offers two parsing models: diff --git a/docs/mkdocs/docs/home/architecture.md b/docs/mkdocs/docs/home/architecture.md index 4975c1d1b..1079c9783 100644 --- a/docs/mkdocs/docs/home/architecture.md +++ b/docs/mkdocs/docs/home/architecture.md @@ -59,7 +59,7 @@ below are relative to `include/nlohmann`. |-----------|----------| | Value type enumeration | [`detail/value_t.hpp`](https://github.com/nlohmann/json/blob/develop/include/nlohmann/detail/value_t.hpp) | | Input adapters | [`detail/input/input_adapters.hpp`](https://github.com/nlohmann/json/blob/develop/include/nlohmann/detail/input/input_adapters.hpp) | -| Lexer | [`detail/input/lexer.hpp`](https://github.com/nlohmann/json/blob/develop/include/nlohmann/detail/input/lexer.hpp), [`detail/input/number_parse.hpp`](https://github.com/nlohmann/json/blob/develop/include/nlohmann/detail/input/number_parse.hpp), [`detail/input/string_scan.hpp`](https://github.com/nlohmann/json/blob/develop/include/nlohmann/detail/input/string_scan.hpp) | +| Lexer | [`detail/input/lexer.hpp`](https://github.com/nlohmann/json/blob/develop/include/nlohmann/detail/input/lexer.hpp), [`detail/input/number_parse.hpp`](https://github.com/nlohmann/json/blob/develop/include/nlohmann/detail/input/number_parse.hpp), [`detail/input/string_scan.hpp`](https://github.com/nlohmann/json/blob/develop/include/nlohmann/detail/input/string_scan.hpp), [`detail/input/simd_scan.hpp`](https://github.com/nlohmann/json/blob/develop/include/nlohmann/detail/input/simd_scan.hpp) | | Parser | [`detail/input/parser.hpp`](https://github.com/nlohmann/json/blob/develop/include/nlohmann/detail/input/parser.hpp) | | SAX interface and DOM builders | [`detail/input/json_sax.hpp`](https://github.com/nlohmann/json/blob/develop/include/nlohmann/detail/input/json_sax.hpp) | | Binary format readers | [`detail/input/binary_reader.hpp`](https://github.com/nlohmann/json/blob/develop/include/nlohmann/detail/input/binary_reader.hpp) | diff --git a/docs/mkdocs/mkdocs.yml b/docs/mkdocs/mkdocs.yml index 7d0df1226..2982ef4d6 100644 --- a/docs/mkdocs/mkdocs.yml +++ b/docs/mkdocs/mkdocs.yml @@ -310,6 +310,7 @@ nav: - 'JSON_NO_AUTOMATIC_UDLS': api/macros/json_no_automatic_udls.md - 'JSON_NOEXCEPTION': api/macros/json_noexception.md - 'JSON_NO_IO': api/macros/json_no_io.md + - 'JSON_NO_SIMD': api/macros/json_no_simd.md - 'JSON_NO_THREAD_LOCAL': api/macros/json_no_thread_local.md - 'JSON_PRECISE_STREAM_POSITION': api/macros/json_precise_stream_position.md - 'JSON_SKIP_LIBRARY_VERSION_CHECK': api/macros/json_skip_library_version_check.md diff --git a/include/nlohmann/detail/conversions/to_chars.hpp b/include/nlohmann/detail/conversions/to_chars.hpp index db50795b5..aaac1a64f 100644 --- a/include/nlohmann/detail/conversions/to_chars.hpp +++ b/include/nlohmann/detail/conversions/to_chars.hpp @@ -23,12 +23,12 @@ #endif // SSE2 (every x86-64 CPU) and NEON (every 64-bit Arm CPU) convert the 16 -// digits of a double at once -#if defined(__x86_64__) || (defined(_M_X64) && !defined(_M_ARM64EC)) +// digits of a double at once (unless JSON_NO_SIMD selects the portable code) +#if !defined(JSON_NO_SIMD) && (defined(__x86_64__) || (defined(_M_X64) && !defined(_M_ARM64EC))) #include #define JSON_DTOA_SSE2 1 #define JSON_DTOA_NEON 0 -#elif (defined(__aarch64__) || defined(_M_ARM64)) && !defined(_M_ARM64EC) && !defined(__ARM_BIG_ENDIAN) +#elif !defined(JSON_NO_SIMD) && (defined(__aarch64__) || defined(_M_ARM64)) && !defined(_M_ARM64EC) && !defined(__ARM_BIG_ENDIAN) #include #define JSON_DTOA_SSE2 0 #define JSON_DTOA_NEON 1 diff --git a/include/nlohmann/detail/input/simd_scan.hpp b/include/nlohmann/detail/input/simd_scan.hpp new file mode 100644 index 000000000..69a7f92cc --- /dev/null +++ b/include/nlohmann/detail/input/simd_scan.hpp @@ -0,0 +1,314 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-FileCopyrightText: 2018-2025 The simdjson authors +// SPDX-License-Identifier: MIT + +#pragma once + +#include // array +#include // atomic +#include // size_t +#include // uint8_t, uint64_t + +#include +#include + +// Vector code for runs of string bytes, used by the lexer for contiguous input +// (see string_bulk_run()). NEON (AArch64) and SSE2 (x86-64) belong to the +// baseline instruction sets and are used by default. The vector UTF-8 check +// needs NEON or SSSE3. SSSE3 is not part of x86-64, and the code must not +// depend on the flags of a translation unit (two translation units with +// different flags would have different definitions of the same inline +// functions): the check is compiled for SSSE3 with a function attribute and +// used where the CPU has SSSE3 (all x86-64 CPUs since about 2011), else the +// portable check. Code compiled for SSSE3 anyway (-mssse3 or more) skips the +// CPU check. JSON_NO_SIMD selects the portable code everywhere. +// NLOHMANN_JSON_NO_TARGET_ATTRIBUTE (for compilers that ignore the target +// attribute and then reject the SSSE3 intrinsics) keeps the check portable +// without disabling the SSE2 code. +#if !defined(JSON_NO_SIMD) && defined(__aarch64__) && (defined(__GNUC__) || defined(__clang__)) && defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + #include + #define JSON_SCAN_NEON 1 +#else + #define JSON_SCAN_NEON 0 +#endif +// (x86 only: other targets can define __SSE2__ as well, e.g., WebAssembly with -msse2, but have no ) +#if !defined(JSON_NO_SIMD) && !JSON_SCAN_NEON && (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86)) && !defined(_M_ARM64EC) && (defined(__SSE2__) || defined(_M_X64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2)) + #include + #define JSON_SCAN_SSE2 1 +#else + #define JSON_SCAN_SSE2 0 +#endif +#if JSON_SCAN_SSE2 && defined(__SSSE3__) + #include + #define JSON_SCAN_SSSE3 1 // NOLINT(cppcoreguidelines-macro-to-enum,modernize-macro-to-enum) +#else + #define JSON_SCAN_SSSE3 0 // NOLINT(cppcoreguidelines-macro-to-enum,modernize-macro-to-enum) +#endif +#if JSON_SCAN_SSE2 && !JSON_SCAN_SSSE3 && !defined(NLOHMANN_JSON_NO_TARGET_ATTRIBUTE) && ((defined(__clang__) && __clang_major__ >= 4) || (defined(__GNUC__) && !defined(__clang__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 9)))) + // (GCC before 4.9 has no SSSE3 intrinsics without -mssse3) + #include + #include + #define JSON_SCAN_SSSE3_DISPATCH 1 // NOLINT(cppcoreguidelines-macro-to-enum,modernize-macro-to-enum) + #define JSON_SCAN_SSSE3_TARGET __attribute__((target("ssse3"))) +#elif JSON_SCAN_SSE2 && !JSON_SCAN_SSSE3 && defined(_MSC_VER) + // (MSVC compiles intrinsics of any instruction set) + #include + #include + #define JSON_SCAN_SSSE3_DISPATCH 1 // NOLINT(cppcoreguidelines-macro-to-enum,modernize-macro-to-enum) + #define JSON_SCAN_SSSE3_TARGET +#else + #define JSON_SCAN_SSSE3_DISPATCH 0 // NOLINT(cppcoreguidelines-macro-to-enum,modernize-macro-to-enum) + #define JSON_SCAN_SSSE3_TARGET +#endif +#define JSON_SCAN_VECTOR (JSON_SCAN_NEON || JSON_SCAN_SSE2) +#define JSON_SCAN_VECTOR_UTF8 (JSON_SCAN_NEON || JSON_SCAN_SSSE3 || JSON_SCAN_SSSE3_DISPATCH) + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +#if JSON_SCAN_VECTOR +/*! +@brief the first byte of a string run that is a quote, a backslash, a control +character, or not ASCII, 16 bytes per step + +Stops at such a byte, or where fewer than 16 bytes are left (the caller tells +the two apart). A signed compare with 0x20 finds control characters and +non-ASCII bytes at once. +*/ +JSON_HEDLEY_ALWAYS_INLINE const unsigned char* vector_plain_run(const unsigned char* p, const unsigned char* e) noexcept +{ + while (e - p >= 16) + { +#if JSON_SCAN_NEON + const uint8x16_t in = vld1q_u8(p); + const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('"')), vceqq_u8(in, vdupq_n_u8('\\'))), + vcltq_s8(vreinterpretq_s8_u8(in), vdupq_n_s8(0x20))); + // one nibble per byte (the usual NEON replacement of x86's movemask, see + // D. Kutenin, "Porting x86 vector bitmask optimizations to Arm NEON", 2022) + const std::uint64_t bits = vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(special), 4)), 0); + if (bits != 0) + { + return p + (count_trailing_zeros(bits) >> 2u); + } +#else + const __m128i in = _mm_loadu_si128(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) + const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), + _mm_cmplt_epi8(in, _mm_set1_epi8(0x20))); + const auto bits = static_cast(static_cast(_mm_movemask_epi8(special))); + if (bits != 0) + { + return p + count_trailing_zeros(bits); + } +#endif + p += 16; + } + return p; +} +#endif + +#if JSON_SCAN_VECTOR +/// vector_plain_run() for runs that are already longer than a few words: +/// inline with SSE2, out of line with NEON, where a larger string loop slows +/// down the rest of the lexer +#if JSON_SCAN_NEON + JSON_HEDLEY_NEVER_INLINE inline +#else + JSON_HEDLEY_ALWAYS_INLINE +#endif +const unsigned char* vector_plain_run_long(const unsigned char* p, const unsigned char* e) noexcept +{ + return vector_plain_run(p, e); +} +#endif + +#if JSON_SCAN_SSSE3_DISPATCH +/// whether the CPU has SSSE3 (CPUID leaf 1, ECX bit 9) +inline bool cpu_ssse3() noexcept +{ +#if defined(_MSC_VER) && !defined(__clang__) + std::array regs {{}}; + __cpuid(regs.data(), 1); + return (static_cast(regs[2]) & (1u << 9u)) != 0; +#else + unsigned eax = 0; + unsigned ebx = 0; + unsigned ecx = 0; + unsigned edx = 0; + return __get_cpuid(1, &eax, &ebx, &ecx, &edx) != 0 && (ecx & (1u << 9u)) != 0; +#endif +} + +/// whether the CPU has SSSE3, asked once: the answer is kept in an atomic +/// that is initialized at compile time, so that neither a guard of a local +/// static nor a global constructor is needed (threads that ask at the same +/// time all store the same answer) +JSON_HEDLEY_ALWAYS_INLINE bool cpu_has_ssse3() noexcept +{ + static std::atomic known{0}; // 0: not asked yet, 1: no, 2: yes + int state = known.load(std::memory_order_relaxed); + if (JSON_HEDLEY_UNLIKELY(state == 0)) + { + state = cpu_ssse3() ? 2 : 1; + known.store(state, std::memory_order_relaxed); + } + return state == 2; +} +#endif + +#if JSON_SCAN_VECTOR_UTF8 +/// Tables of the UTF-8 check of J. Keiser and D. Lemire, "Validating UTF-8 In +/// Less Than One Instruction Per Byte" (2021), as in simdjson ("lookup4"): each +/// maps a nibble (high and low nibble of the previous byte, high nibble of the +/// current byte) to the errors it allows; a byte pair is ill-formed if all +/// three have an error bit in common. +template +struct utf8_lookup4 +{ + static constexpr std::uint8_t too_short = 1u << 0u, too_long = 1u << 1u, overlong_3 = 1u << 2u, too_large = 1u << 3u; + static constexpr std::uint8_t surrogate = 1u << 4u, overlong_2 = 1u << 5u, too_large_1000 = 1u << 6u, overlong_4 = 1u << 6u; + static constexpr std::uint8_t two_conts = 1u << 7u, carry = too_short | too_long | two_conts; + static const std::array byte_1_high; + static const std::array byte_1_low; + static const std::array byte_2_high; +}; + +template +const std::array utf8_lookup4::byte_1_high = +{ + { + too_long, too_long, too_long, too_long, too_long, too_long, too_long, too_long, + two_conts, two_conts, two_conts, two_conts, + too_short | overlong_2, too_short, too_short | overlong_3 | surrogate, too_short | too_large | too_large_1000 | overlong_4 + } +}; + +template +const std::array utf8_lookup4::byte_1_low = +{ + { + carry | overlong_3 | overlong_2 | overlong_4, carry | overlong_2, carry, carry, + carry | too_large, carry | too_large | too_large_1000, carry | too_large | too_large_1000, carry | too_large | too_large_1000, + carry | too_large | too_large_1000, carry | too_large | too_large_1000, carry | too_large | too_large_1000, carry | too_large | too_large_1000, + carry | too_large | too_large_1000, carry | too_large | too_large_1000 | surrogate, carry | too_large | too_large_1000, carry | too_large | too_large_1000 + } +}; + +template +const std::array utf8_lookup4::byte_2_high = +{ + { + too_short, too_short, too_short, too_short, too_short, too_short, too_short, too_short, + static_cast(too_long | overlong_2 | two_conts | overlong_3 | too_large_1000 | overlong_4), + static_cast(too_long | overlong_2 | two_conts | overlong_3 | too_large), + static_cast(too_long | overlong_2 | two_conts | surrogate | too_large), + static_cast(too_long | overlong_2 | two_conts | surrogate | too_large), + too_short, too_short, too_short, too_short + } +}; + +/*! +@brief the run of plain string bytes and well-formed UTF-8 at p (a character + boundary), 16 bytes per step + +The first quote, backslash, or control character is found with vector +compares, and the UTF-8 check of J. Keiser and D. Lemire covers the bytes up +to it. Returns true with @a block at that byte; returns false with @a block at +the start of the 16 bytes where the check found ill-formed UTF-8 or where +fewer than 16 bytes are left, for the caller to finish one sequence at a time. +Out of line, so that no constants of the check occupy registers in the +caller's loop. On x86-64, it is compiled for SSSE3 (see cpu_has_ssse3()). +*/ +JSON_SCAN_SSSE3_TARGET JSON_HEDLEY_NEVER_INLINE inline bool vector_utf8_run(const unsigned char* p, const unsigned char* e, const unsigned char*& block_out) noexcept +{ + using lookup = utf8_lookup4<>; + const unsigned char* block = p; +#if JSON_SCAN_NEON + const uint8x16_t t1h = vld1q_u8(lookup::byte_1_high.data()); + const uint8x16_t t1l = vld1q_u8(lookup::byte_1_low.data()); + const uint8x16_t t2h = vld1q_u8(lookup::byte_2_high.data()); + uint8x16_t prev = vdupq_n_u8(0); + while (e - block >= 16) + { + const uint8x16_t in = vld1q_u8(block); + const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('"')), vceqq_u8(in, vdupq_n_u8('\\'))), vcltq_u8(in, vdupq_n_u8(0x20))); + const uint8x16_t prev1 = vextq_u8(prev, in, 15); + const uint8x16_t sc = vandq_u8(vandq_u8(vqtbl1q_u8(t1h, vshrq_n_u8(prev1, 4)), vqtbl1q_u8(t1l, vandq_u8(prev1, vdupq_n_u8(0x0F)))), vqtbl1q_u8(t2h, vshrq_n_u8(in, 4))); + const uint8x16_t must23 = vorrq_u8(vqsubq_u8(vextq_u8(prev, in, 14), vdupq_n_u8(0xE0 - 0x80)), vqsubq_u8(vextq_u8(prev, in, 13), vdupq_n_u8(0xF0 - 0x80))); + const uint8x16_t err = veorq_u8(vandq_u8(must23, vdupq_n_u8(0x80)), sc); + const std::uint64_t special_bits = vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(special), 4)), 0); + const std::uint64_t err_bits = vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(vtstq_u8(err, err)), 4)), 0); + if (special_bits != 0) + { + // errors up to the special byte count (an incomplete sequence + // before a quote shows at the quote); the bytes after it do not + const unsigned k = static_cast(count_trailing_zeros(special_bits)) >> 2u; + const std::uint64_t upto = k == 15 ? ~std::uint64_t{0} : + (std::uint64_t{1} << (4u * (k + 1u))) - 1u; + if ((err_bits & upto) == 0) + { + block_out = block + k; + return true; + } + break; + } + if (err_bits != 0) + { + break; + } + prev = in; + block += 16; + } +#else + // the same with SSSE3 (pshufb for the table lookups; nibbles from 16-bit + // shifts, as there are no byte shifts) + const __m128i t1h = _mm_loadu_si128(static_cast(static_cast(lookup::byte_1_high.data()))); // NOLINT(bugprone-casting-through-void) + const __m128i t1l = _mm_loadu_si128(static_cast(static_cast(lookup::byte_1_low.data()))); // NOLINT(bugprone-casting-through-void) + const __m128i t2h = _mm_loadu_si128(static_cast(static_cast(lookup::byte_2_high.data()))); // NOLINT(bugprone-casting-through-void) + const __m128i nibble = _mm_set1_epi8(0x0F); + const __m128i zero = _mm_setzero_si128(); + __m128i prev = zero; + while (e - block >= 16) + { + const __m128i in = _mm_loadu_si128(static_cast(static_cast(block))); // NOLINT(bugprone-casting-through-void) + const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), + _mm_cmpeq_epi8(_mm_subs_epu8(in, _mm_set1_epi8(0x1F)), zero)); // in < 0x20 + const __m128i prev1 = _mm_alignr_epi8(in, prev, 15); + const __m128i sc = _mm_and_si128(_mm_and_si128(_mm_shuffle_epi8(t1h, _mm_and_si128(_mm_srli_epi16(prev1, 4), nibble)), + _mm_shuffle_epi8(t1l, _mm_and_si128(prev1, nibble))), + _mm_shuffle_epi8(t2h, _mm_and_si128(_mm_srli_epi16(in, 4), nibble))); + const __m128i must23 = _mm_or_si128(_mm_subs_epu8(_mm_alignr_epi8(in, prev, 14), _mm_set1_epi8(0xE0 - 0x80)), + _mm_subs_epu8(_mm_alignr_epi8(in, prev, 13), _mm_set1_epi8(0xF0 - 0x80))); + const __m128i err = _mm_xor_si128(_mm_and_si128(must23, _mm_set1_epi8(static_cast(-128))), sc); + const auto special_bits = static_cast(_mm_movemask_epi8(special)); + const auto err_bits = ~static_cast(_mm_movemask_epi8(_mm_cmpeq_epi8(err, zero))) & 0xFFFFu; + if (special_bits != 0) + { + const auto k = static_cast(count_trailing_zeros(static_cast(special_bits))); + if ((err_bits & ((2u << k) - 1u)) == 0) + { + block_out = block + k; + return true; + } + break; + } + if (err_bits != 0) + { + break; + } + prev = in; + block += 16; + } +#endif + block_out = block; + return false; +} +#endif + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END diff --git a/include/nlohmann/detail/input/string_scan.hpp b/include/nlohmann/detail/input/string_scan.hpp index fd2289b47..d41005fcd 100644 --- a/include/nlohmann/detail/input/string_scan.hpp +++ b/include/nlohmann/detail/input/string_scan.hpp @@ -14,6 +14,7 @@ #include // memcpy #include +#include #include // Optional SIMD backend for bulk UTF-8 validation. This is an opt-in external @@ -66,7 +67,7 @@ inline std::uint64_t swar_string_special(std::uint64_t v) noexcept // return the index of the first is_string_special() byte in [data, data+n), or // n if every byte is ordinary; scans 8 bytes at a time -inline std::size_t find_string_special(const unsigned char* data, std::size_t n) noexcept +JSON_HEDLEY_ALWAYS_INLINE std::size_t find_string_special(const unsigned char* data, std::size_t n) noexcept { std::size_t i = 0; for (; i + 8 <= n; i += 8) @@ -235,35 +236,136 @@ inline std::size_t valid_utf8_prefix(const unsigned char* data, std::size_t n) n return pos; } -// Scalar (C++11) computation of the bulk run length: the number of leading -// bytes in [data, data+n) that are ordinary ASCII or complete well-formed UTF-8 +#if JSON_SCAN_VECTOR_UTF8 +/// the end of scan_string_vector from block, where the vector loop stopped +/// (ill-formed UTF-8, or fewer than 16 bytes left): one byte or sequence at a +/// time, from the start of a sequence that crosses into the block +inline const unsigned char* scan_string_finish(const unsigned char* p, const unsigned char* block, const unsigned char* e) noexcept +{ + for (int i = 1; i <= 3 && block - i >= p; ++i) + { + const unsigned char c = block[-i]; + if (c < 0x80) + { + break; + } + if (c >= 0xC0) + { + const int len = 2 + static_cast(c >= 0xE0) + static_cast(c >= 0xF0); + if (len > i) + { + block -= i; + } + break; + } + } + for (p = block; p != e;) + { + if (*p < 0x80) + { + if (is_string_special(*p)) + { + return p; + } + ++p; + continue; + } + const std::size_t n = validate_one_utf8(p, static_cast(e - p)); + if (n == 0) + { + return p; + } + p += n; + } + return p; +} + +/// the rest of a string from p (a character boundary), with the vector UTF-8 +/// check and the scalar check for the end of the block where it stopped +inline const unsigned char* scan_string_vector(const unsigned char* p, const unsigned char* e) noexcept +{ + const unsigned char* block = p; + if (vector_utf8_run(p, e, block)) + { + return block; + } + return scan_string_finish(p, block, e); +} +#endif + +#if JSON_SCAN_VECTOR +/// the end of the run of plain string characters at @a p: the first quote, +/// backslash, control character, or non-ASCII byte, or @a e. The first words +/// are checked with SWAR (one; four with NEON, where the vector mask costs +/// more), as most keys and short values end there; longer runs continue 16 +/// bytes at a time. +JSON_HEDLEY_ALWAYS_INLINE const unsigned char* plain_run_end(const unsigned char* p, const unsigned char* e) noexcept +{ + constexpr int prefix_words = JSON_SCAN_NEON ? 4 : 1; + for (int words = prefix_words; words > 0 && e - p >= 8; --words) + { + const std::uint64_t special = swar_string_special(read_eight_bytes(p)); + if (special != 0) + { + return p + (static_cast(count_trailing_zeros(special)) / 8); + } + p += 8; + } + p = vector_plain_run_long(p, e); + if (e - p < 16) // else the vector loop stopped at a special byte + { + p += find_string_special(p, static_cast(e - p)); + } + return p; +} +#endif + +// Computation of the bulk run length: the number of leading bytes in +// [data, data+n) that are ordinary ASCII or complete well-formed UTF-8 // sequences, stopping before the first byte that needs individual handling (the // closing quote, an escape, a control character, or an ill-formed/truncated -// sequence). ASCII is skipped 8 bytes at a time. -inline std::size_t scalar_string_bulk_run(const unsigned char* data, std::size_t n) noexcept +// sequence). ASCII is skipped 16 bytes at a time with SSE2 and 8 bytes at a +// time otherwise; runs of non-ASCII text use the vector UTF-8 check where +// available. +JSON_HEDLEY_ALWAYS_INLINE std::size_t scalar_string_bulk_run(const unsigned char* data, std::size_t n) noexcept { - std::size_t pos = 0; - while (pos < n) + const unsigned char* const e = data + n; + const unsigned char* p = data; + while (p != e) { - pos += find_string_special(data + pos, n - pos); - if (pos >= n || data[pos] < 0x80u) +#if JSON_SCAN_VECTOR + p = plain_run_end(p, e); +#else + p += find_string_special(p, static_cast(e - p)); +#endif + if (p == e || *p < 0x80u) { break; // end of buffer, or a quote/escape/control byte } +#if JSON_SCAN_VECTOR_UTF8 +#if JSON_SCAN_SSSE3_DISPATCH + if (JSON_HEDLEY_LIKELY(cpu_has_ssse3())) +#endif + { + return static_cast(scan_string_vector(p, e) - data); + } +#endif +#if !JSON_SCAN_VECTOR_UTF8 || JSON_SCAN_SSSE3_DISPATCH // a run of multi-byte sequences (e.g. CJK text) is validated sequence // by sequence without searching for the next special byte in between do { - const std::size_t seq = validate_one_utf8(data + pos, n - pos); + const std::size_t seq = validate_one_utf8(p, static_cast(e - p)); if (seq == 0) { - return pos; // ill-formed or truncated: let the byte path diagnose it + return static_cast(p - data); // ill-formed or truncated: let the byte path diagnose it } - pos += seq; + p += seq; } - while (pos < n && data[pos] >= 0x80u); + while (p != e && *p >= 0x80u); +#endif } - return pos; + return static_cast(p - data); } #if defined(JSON_USE_SIMDUTF) && defined(JSON_HAS_CPP_17) @@ -304,7 +406,7 @@ inline std::size_t find_string_delimiter(const unsigned char* data, std::size_t // next delimiter is validated in one shot by simdutf; on the rare failure the // scalar helper recomputes the exact valid prefix so the byte path still // produces the precise diagnostic. Without it, the pure scalar path is used. -inline std::size_t string_bulk_run(const unsigned char* data, std::size_t n) noexcept +JSON_HEDLEY_ALWAYS_INLINE std::size_t string_bulk_run(const unsigned char* data, std::size_t n) noexcept { #if defined(JSON_USE_SIMDUTF) && defined(JSON_HAS_CPP_17) const std::size_t run = find_string_delimiter(data, n); diff --git a/include/nlohmann/detail/macro_unscope.hpp b/include/nlohmann/detail/macro_unscope.hpp index f3775aa0c..426454510 100644 --- a/include/nlohmann/detail/macro_unscope.hpp +++ b/include/nlohmann/detail/macro_unscope.hpp @@ -26,6 +26,13 @@ #undef JSON_UNLOCKED_FILE_IO #undef JSON_DTOA_SSE2 #undef JSON_DTOA_NEON +#undef JSON_SCAN_NEON +#undef JSON_SCAN_SSE2 +#undef JSON_SCAN_SSSE3 +#undef JSON_SCAN_SSSE3_DISPATCH +#undef JSON_SCAN_SSSE3_TARGET +#undef JSON_SCAN_VECTOR +#undef JSON_SCAN_VECTOR_UTF8 #ifndef JSON_TEST_KEEP_MACROS #undef JSON_CATCH diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 9a4e2b2d4..6c32488e5 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -10834,6 +10834,324 @@ NLOHMANN_JSON_NAMESPACE_END // #include +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-FileCopyrightText: 2018-2025 The simdjson authors +// SPDX-License-Identifier: MIT + + + +#include // array +#include // atomic +#include // size_t +#include // uint8_t, uint64_t + +// #include + +// #include + + +// Vector code for runs of string bytes, used by the lexer for contiguous input +// (see string_bulk_run()). NEON (AArch64) and SSE2 (x86-64) belong to the +// baseline instruction sets and are used by default. The vector UTF-8 check +// needs NEON or SSSE3. SSSE3 is not part of x86-64, and the code must not +// depend on the flags of a translation unit (two translation units with +// different flags would have different definitions of the same inline +// functions): the check is compiled for SSSE3 with a function attribute and +// used where the CPU has SSSE3 (all x86-64 CPUs since about 2011), else the +// portable check. Code compiled for SSSE3 anyway (-mssse3 or more) skips the +// CPU check. JSON_NO_SIMD selects the portable code everywhere. +// NLOHMANN_JSON_NO_TARGET_ATTRIBUTE (for compilers that ignore the target +// attribute and then reject the SSSE3 intrinsics) keeps the check portable +// without disabling the SSE2 code. +#if !defined(JSON_NO_SIMD) && defined(__aarch64__) && (defined(__GNUC__) || defined(__clang__)) && defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + #include + #define JSON_SCAN_NEON 1 +#else + #define JSON_SCAN_NEON 0 +#endif +// (x86 only: other targets can define __SSE2__ as well, e.g., WebAssembly with -msse2, but have no ) +#if !defined(JSON_NO_SIMD) && !JSON_SCAN_NEON && (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86)) && !defined(_M_ARM64EC) && (defined(__SSE2__) || defined(_M_X64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2)) + #include + #define JSON_SCAN_SSE2 1 +#else + #define JSON_SCAN_SSE2 0 +#endif +#if JSON_SCAN_SSE2 && defined(__SSSE3__) + #include + #define JSON_SCAN_SSSE3 1 // NOLINT(cppcoreguidelines-macro-to-enum,modernize-macro-to-enum) +#else + #define JSON_SCAN_SSSE3 0 // NOLINT(cppcoreguidelines-macro-to-enum,modernize-macro-to-enum) +#endif +#if JSON_SCAN_SSE2 && !JSON_SCAN_SSSE3 && !defined(NLOHMANN_JSON_NO_TARGET_ATTRIBUTE) && ((defined(__clang__) && __clang_major__ >= 4) || (defined(__GNUC__) && !defined(__clang__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 9)))) + // (GCC before 4.9 has no SSSE3 intrinsics without -mssse3) + #include + #include + #define JSON_SCAN_SSSE3_DISPATCH 1 // NOLINT(cppcoreguidelines-macro-to-enum,modernize-macro-to-enum) + #define JSON_SCAN_SSSE3_TARGET __attribute__((target("ssse3"))) +#elif JSON_SCAN_SSE2 && !JSON_SCAN_SSSE3 && defined(_MSC_VER) + // (MSVC compiles intrinsics of any instruction set) + #include + #include + #define JSON_SCAN_SSSE3_DISPATCH 1 // NOLINT(cppcoreguidelines-macro-to-enum,modernize-macro-to-enum) + #define JSON_SCAN_SSSE3_TARGET +#else + #define JSON_SCAN_SSSE3_DISPATCH 0 // NOLINT(cppcoreguidelines-macro-to-enum,modernize-macro-to-enum) + #define JSON_SCAN_SSSE3_TARGET +#endif +#define JSON_SCAN_VECTOR (JSON_SCAN_NEON || JSON_SCAN_SSE2) +#define JSON_SCAN_VECTOR_UTF8 (JSON_SCAN_NEON || JSON_SCAN_SSSE3 || JSON_SCAN_SSSE3_DISPATCH) + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +#if JSON_SCAN_VECTOR +/*! +@brief the first byte of a string run that is a quote, a backslash, a control +character, or not ASCII, 16 bytes per step + +Stops at such a byte, or where fewer than 16 bytes are left (the caller tells +the two apart). A signed compare with 0x20 finds control characters and +non-ASCII bytes at once. +*/ +JSON_HEDLEY_ALWAYS_INLINE const unsigned char* vector_plain_run(const unsigned char* p, const unsigned char* e) noexcept +{ + while (e - p >= 16) + { +#if JSON_SCAN_NEON + const uint8x16_t in = vld1q_u8(p); + const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('"')), vceqq_u8(in, vdupq_n_u8('\\'))), + vcltq_s8(vreinterpretq_s8_u8(in), vdupq_n_s8(0x20))); + // one nibble per byte (the usual NEON replacement of x86's movemask, see + // D. Kutenin, "Porting x86 vector bitmask optimizations to Arm NEON", 2022) + const std::uint64_t bits = vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(special), 4)), 0); + if (bits != 0) + { + return p + (count_trailing_zeros(bits) >> 2u); + } +#else + const __m128i in = _mm_loadu_si128(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) + const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), + _mm_cmplt_epi8(in, _mm_set1_epi8(0x20))); + const auto bits = static_cast(static_cast(_mm_movemask_epi8(special))); + if (bits != 0) + { + return p + count_trailing_zeros(bits); + } +#endif + p += 16; + } + return p; +} +#endif + +#if JSON_SCAN_VECTOR +/// vector_plain_run() for runs that are already longer than a few words: +/// inline with SSE2, out of line with NEON, where a larger string loop slows +/// down the rest of the lexer +#if JSON_SCAN_NEON + JSON_HEDLEY_NEVER_INLINE inline +#else + JSON_HEDLEY_ALWAYS_INLINE +#endif +const unsigned char* vector_plain_run_long(const unsigned char* p, const unsigned char* e) noexcept +{ + return vector_plain_run(p, e); +} +#endif + +#if JSON_SCAN_SSSE3_DISPATCH +/// whether the CPU has SSSE3 (CPUID leaf 1, ECX bit 9) +inline bool cpu_ssse3() noexcept +{ +#if defined(_MSC_VER) && !defined(__clang__) + std::array regs {{}}; + __cpuid(regs.data(), 1); + return (static_cast(regs[2]) & (1u << 9u)) != 0; +#else + unsigned eax = 0; + unsigned ebx = 0; + unsigned ecx = 0; + unsigned edx = 0; + return __get_cpuid(1, &eax, &ebx, &ecx, &edx) != 0 && (ecx & (1u << 9u)) != 0; +#endif +} + +/// whether the CPU has SSSE3, asked once: the answer is kept in an atomic +/// that is initialized at compile time, so that neither a guard of a local +/// static nor a global constructor is needed (threads that ask at the same +/// time all store the same answer) +JSON_HEDLEY_ALWAYS_INLINE bool cpu_has_ssse3() noexcept +{ + static std::atomic known{0}; // 0: not asked yet, 1: no, 2: yes + int state = known.load(std::memory_order_relaxed); + if (JSON_HEDLEY_UNLIKELY(state == 0)) + { + state = cpu_ssse3() ? 2 : 1; + known.store(state, std::memory_order_relaxed); + } + return state == 2; +} +#endif + +#if JSON_SCAN_VECTOR_UTF8 +/// Tables of the UTF-8 check of J. Keiser and D. Lemire, "Validating UTF-8 In +/// Less Than One Instruction Per Byte" (2021), as in simdjson ("lookup4"): each +/// maps a nibble (high and low nibble of the previous byte, high nibble of the +/// current byte) to the errors it allows; a byte pair is ill-formed if all +/// three have an error bit in common. +template +struct utf8_lookup4 +{ + static constexpr std::uint8_t too_short = 1u << 0u, too_long = 1u << 1u, overlong_3 = 1u << 2u, too_large = 1u << 3u; + static constexpr std::uint8_t surrogate = 1u << 4u, overlong_2 = 1u << 5u, too_large_1000 = 1u << 6u, overlong_4 = 1u << 6u; + static constexpr std::uint8_t two_conts = 1u << 7u, carry = too_short | too_long | two_conts; + static const std::array byte_1_high; + static const std::array byte_1_low; + static const std::array byte_2_high; +}; + +template +const std::array utf8_lookup4::byte_1_high = +{ + { + too_long, too_long, too_long, too_long, too_long, too_long, too_long, too_long, + two_conts, two_conts, two_conts, two_conts, + too_short | overlong_2, too_short, too_short | overlong_3 | surrogate, too_short | too_large | too_large_1000 | overlong_4 + } +}; + +template +const std::array utf8_lookup4::byte_1_low = +{ + { + carry | overlong_3 | overlong_2 | overlong_4, carry | overlong_2, carry, carry, + carry | too_large, carry | too_large | too_large_1000, carry | too_large | too_large_1000, carry | too_large | too_large_1000, + carry | too_large | too_large_1000, carry | too_large | too_large_1000, carry | too_large | too_large_1000, carry | too_large | too_large_1000, + carry | too_large | too_large_1000, carry | too_large | too_large_1000 | surrogate, carry | too_large | too_large_1000, carry | too_large | too_large_1000 + } +}; + +template +const std::array utf8_lookup4::byte_2_high = +{ + { + too_short, too_short, too_short, too_short, too_short, too_short, too_short, too_short, + static_cast(too_long | overlong_2 | two_conts | overlong_3 | too_large_1000 | overlong_4), + static_cast(too_long | overlong_2 | two_conts | overlong_3 | too_large), + static_cast(too_long | overlong_2 | two_conts | surrogate | too_large), + static_cast(too_long | overlong_2 | two_conts | surrogate | too_large), + too_short, too_short, too_short, too_short + } +}; + +/*! +@brief the run of plain string bytes and well-formed UTF-8 at p (a character + boundary), 16 bytes per step + +The first quote, backslash, or control character is found with vector +compares, and the UTF-8 check of J. Keiser and D. Lemire covers the bytes up +to it. Returns true with @a block at that byte; returns false with @a block at +the start of the 16 bytes where the check found ill-formed UTF-8 or where +fewer than 16 bytes are left, for the caller to finish one sequence at a time. +Out of line, so that no constants of the check occupy registers in the +caller's loop. On x86-64, it is compiled for SSSE3 (see cpu_has_ssse3()). +*/ +JSON_SCAN_SSSE3_TARGET JSON_HEDLEY_NEVER_INLINE inline bool vector_utf8_run(const unsigned char* p, const unsigned char* e, const unsigned char*& block_out) noexcept +{ + using lookup = utf8_lookup4<>; + const unsigned char* block = p; +#if JSON_SCAN_NEON + const uint8x16_t t1h = vld1q_u8(lookup::byte_1_high.data()); + const uint8x16_t t1l = vld1q_u8(lookup::byte_1_low.data()); + const uint8x16_t t2h = vld1q_u8(lookup::byte_2_high.data()); + uint8x16_t prev = vdupq_n_u8(0); + while (e - block >= 16) + { + const uint8x16_t in = vld1q_u8(block); + const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('"')), vceqq_u8(in, vdupq_n_u8('\\'))), vcltq_u8(in, vdupq_n_u8(0x20))); + const uint8x16_t prev1 = vextq_u8(prev, in, 15); + const uint8x16_t sc = vandq_u8(vandq_u8(vqtbl1q_u8(t1h, vshrq_n_u8(prev1, 4)), vqtbl1q_u8(t1l, vandq_u8(prev1, vdupq_n_u8(0x0F)))), vqtbl1q_u8(t2h, vshrq_n_u8(in, 4))); + const uint8x16_t must23 = vorrq_u8(vqsubq_u8(vextq_u8(prev, in, 14), vdupq_n_u8(0xE0 - 0x80)), vqsubq_u8(vextq_u8(prev, in, 13), vdupq_n_u8(0xF0 - 0x80))); + const uint8x16_t err = veorq_u8(vandq_u8(must23, vdupq_n_u8(0x80)), sc); + const std::uint64_t special_bits = vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(special), 4)), 0); + const std::uint64_t err_bits = vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(vtstq_u8(err, err)), 4)), 0); + if (special_bits != 0) + { + // errors up to the special byte count (an incomplete sequence + // before a quote shows at the quote); the bytes after it do not + const unsigned k = static_cast(count_trailing_zeros(special_bits)) >> 2u; + const std::uint64_t upto = k == 15 ? ~std::uint64_t{0} : + (std::uint64_t{1} << (4u * (k + 1u))) - 1u; + if ((err_bits & upto) == 0) + { + block_out = block + k; + return true; + } + break; + } + if (err_bits != 0) + { + break; + } + prev = in; + block += 16; + } +#else + // the same with SSSE3 (pshufb for the table lookups; nibbles from 16-bit + // shifts, as there are no byte shifts) + const __m128i t1h = _mm_loadu_si128(static_cast(static_cast(lookup::byte_1_high.data()))); // NOLINT(bugprone-casting-through-void) + const __m128i t1l = _mm_loadu_si128(static_cast(static_cast(lookup::byte_1_low.data()))); // NOLINT(bugprone-casting-through-void) + const __m128i t2h = _mm_loadu_si128(static_cast(static_cast(lookup::byte_2_high.data()))); // NOLINT(bugprone-casting-through-void) + const __m128i nibble = _mm_set1_epi8(0x0F); + const __m128i zero = _mm_setzero_si128(); + __m128i prev = zero; + while (e - block >= 16) + { + const __m128i in = _mm_loadu_si128(static_cast(static_cast(block))); // NOLINT(bugprone-casting-through-void) + const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), + _mm_cmpeq_epi8(_mm_subs_epu8(in, _mm_set1_epi8(0x1F)), zero)); // in < 0x20 + const __m128i prev1 = _mm_alignr_epi8(in, prev, 15); + const __m128i sc = _mm_and_si128(_mm_and_si128(_mm_shuffle_epi8(t1h, _mm_and_si128(_mm_srli_epi16(prev1, 4), nibble)), + _mm_shuffle_epi8(t1l, _mm_and_si128(prev1, nibble))), + _mm_shuffle_epi8(t2h, _mm_and_si128(_mm_srli_epi16(in, 4), nibble))); + const __m128i must23 = _mm_or_si128(_mm_subs_epu8(_mm_alignr_epi8(in, prev, 14), _mm_set1_epi8(0xE0 - 0x80)), + _mm_subs_epu8(_mm_alignr_epi8(in, prev, 13), _mm_set1_epi8(0xF0 - 0x80))); + const __m128i err = _mm_xor_si128(_mm_and_si128(must23, _mm_set1_epi8(static_cast(-128))), sc); + const auto special_bits = static_cast(_mm_movemask_epi8(special)); + const auto err_bits = ~static_cast(_mm_movemask_epi8(_mm_cmpeq_epi8(err, zero))) & 0xFFFFu; + if (special_bits != 0) + { + const auto k = static_cast(count_trailing_zeros(static_cast(special_bits))); + if ((err_bits & ((2u << k) - 1u)) == 0) + { + block_out = block + k; + return true; + } + break; + } + if (err_bits != 0) + { + break; + } + prev = in; + block += 16; + } +#endif + block_out = block; + return false; +} +#endif + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + // #include @@ -10887,7 +11205,7 @@ inline std::uint64_t swar_string_special(std::uint64_t v) noexcept // return the index of the first is_string_special() byte in [data, data+n), or // n if every byte is ordinary; scans 8 bytes at a time -inline std::size_t find_string_special(const unsigned char* data, std::size_t n) noexcept +JSON_HEDLEY_ALWAYS_INLINE std::size_t find_string_special(const unsigned char* data, std::size_t n) noexcept { std::size_t i = 0; for (; i + 8 <= n; i += 8) @@ -11056,35 +11374,136 @@ inline std::size_t valid_utf8_prefix(const unsigned char* data, std::size_t n) n return pos; } -// Scalar (C++11) computation of the bulk run length: the number of leading -// bytes in [data, data+n) that are ordinary ASCII or complete well-formed UTF-8 +#if JSON_SCAN_VECTOR_UTF8 +/// the end of scan_string_vector from block, where the vector loop stopped +/// (ill-formed UTF-8, or fewer than 16 bytes left): one byte or sequence at a +/// time, from the start of a sequence that crosses into the block +inline const unsigned char* scan_string_finish(const unsigned char* p, const unsigned char* block, const unsigned char* e) noexcept +{ + for (int i = 1; i <= 3 && block - i >= p; ++i) + { + const unsigned char c = block[-i]; + if (c < 0x80) + { + break; + } + if (c >= 0xC0) + { + const int len = 2 + static_cast(c >= 0xE0) + static_cast(c >= 0xF0); + if (len > i) + { + block -= i; + } + break; + } + } + for (p = block; p != e;) + { + if (*p < 0x80) + { + if (is_string_special(*p)) + { + return p; + } + ++p; + continue; + } + const std::size_t n = validate_one_utf8(p, static_cast(e - p)); + if (n == 0) + { + return p; + } + p += n; + } + return p; +} + +/// the rest of a string from p (a character boundary), with the vector UTF-8 +/// check and the scalar check for the end of the block where it stopped +inline const unsigned char* scan_string_vector(const unsigned char* p, const unsigned char* e) noexcept +{ + const unsigned char* block = p; + if (vector_utf8_run(p, e, block)) + { + return block; + } + return scan_string_finish(p, block, e); +} +#endif + +#if JSON_SCAN_VECTOR +/// the end of the run of plain string characters at @a p: the first quote, +/// backslash, control character, or non-ASCII byte, or @a e. The first words +/// are checked with SWAR (one; four with NEON, where the vector mask costs +/// more), as most keys and short values end there; longer runs continue 16 +/// bytes at a time. +JSON_HEDLEY_ALWAYS_INLINE const unsigned char* plain_run_end(const unsigned char* p, const unsigned char* e) noexcept +{ + constexpr int prefix_words = JSON_SCAN_NEON ? 4 : 1; + for (int words = prefix_words; words > 0 && e - p >= 8; --words) + { + const std::uint64_t special = swar_string_special(read_eight_bytes(p)); + if (special != 0) + { + return p + (static_cast(count_trailing_zeros(special)) / 8); + } + p += 8; + } + p = vector_plain_run_long(p, e); + if (e - p < 16) // else the vector loop stopped at a special byte + { + p += find_string_special(p, static_cast(e - p)); + } + return p; +} +#endif + +// Computation of the bulk run length: the number of leading bytes in +// [data, data+n) that are ordinary ASCII or complete well-formed UTF-8 // sequences, stopping before the first byte that needs individual handling (the // closing quote, an escape, a control character, or an ill-formed/truncated -// sequence). ASCII is skipped 8 bytes at a time. -inline std::size_t scalar_string_bulk_run(const unsigned char* data, std::size_t n) noexcept +// sequence). ASCII is skipped 16 bytes at a time with SSE2 and 8 bytes at a +// time otherwise; runs of non-ASCII text use the vector UTF-8 check where +// available. +JSON_HEDLEY_ALWAYS_INLINE std::size_t scalar_string_bulk_run(const unsigned char* data, std::size_t n) noexcept { - std::size_t pos = 0; - while (pos < n) + const unsigned char* const e = data + n; + const unsigned char* p = data; + while (p != e) { - pos += find_string_special(data + pos, n - pos); - if (pos >= n || data[pos] < 0x80u) +#if JSON_SCAN_VECTOR + p = plain_run_end(p, e); +#else + p += find_string_special(p, static_cast(e - p)); +#endif + if (p == e || *p < 0x80u) { break; // end of buffer, or a quote/escape/control byte } +#if JSON_SCAN_VECTOR_UTF8 +#if JSON_SCAN_SSSE3_DISPATCH + if (JSON_HEDLEY_LIKELY(cpu_has_ssse3())) +#endif + { + return static_cast(scan_string_vector(p, e) - data); + } +#endif +#if !JSON_SCAN_VECTOR_UTF8 || JSON_SCAN_SSSE3_DISPATCH // a run of multi-byte sequences (e.g. CJK text) is validated sequence // by sequence without searching for the next special byte in between do { - const std::size_t seq = validate_one_utf8(data + pos, n - pos); + const std::size_t seq = validate_one_utf8(p, static_cast(e - p)); if (seq == 0) { - return pos; // ill-formed or truncated: let the byte path diagnose it + return static_cast(p - data); // ill-formed or truncated: let the byte path diagnose it } - pos += seq; + p += seq; } - while (pos < n && data[pos] >= 0x80u); + while (p != e && *p >= 0x80u); +#endif } - return pos; + return static_cast(p - data); } #if defined(JSON_USE_SIMDUTF) && defined(JSON_HAS_CPP_17) @@ -11125,7 +11544,7 @@ inline std::size_t find_string_delimiter(const unsigned char* data, std::size_t // next delimiter is validated in one shot by simdutf; on the rare failure the // scalar helper recomputes the exact valid prefix so the byte path still // produces the precise diagnostic. Without it, the pure scalar path is used. -inline std::size_t string_bulk_run(const unsigned char* data, std::size_t n) noexcept +JSON_HEDLEY_ALWAYS_INLINE std::size_t string_bulk_run(const unsigned char* data, std::size_t n) noexcept { #if defined(JSON_USE_SIMDUTF) && defined(JSON_HAS_CPP_17) const std::size_t run = find_string_delimiter(data, n); @@ -26274,12 +26693,12 @@ NLOHMANN_JSON_NAMESPACE_END #endif // SSE2 (every x86-64 CPU) and NEON (every 64-bit Arm CPU) convert the 16 -// digits of a double at once -#if defined(__x86_64__) || (defined(_M_X64) && !defined(_M_ARM64EC)) +// digits of a double at once (unless JSON_NO_SIMD selects the portable code) +#if !defined(JSON_NO_SIMD) && (defined(__x86_64__) || (defined(_M_X64) && !defined(_M_ARM64EC))) #include #define JSON_DTOA_SSE2 1 #define JSON_DTOA_NEON 0 -#elif (defined(__aarch64__) || defined(_M_ARM64)) && !defined(_M_ARM64EC) && !defined(__ARM_BIG_ENDIAN) +#elif !defined(JSON_NO_SIMD) && (defined(__aarch64__) || defined(_M_ARM64)) && !defined(_M_ARM64EC) && !defined(__ARM_BIG_ENDIAN) #include #define JSON_DTOA_SSE2 0 #define JSON_DTOA_NEON 1 @@ -37774,6 +38193,13 @@ struct formatter // NOLINT(cert-dcl58-c #undef JSON_UNLOCKED_FILE_IO #undef JSON_DTOA_SSE2 #undef JSON_DTOA_NEON +#undef JSON_SCAN_NEON +#undef JSON_SCAN_SSE2 +#undef JSON_SCAN_SSSE3 +#undef JSON_SCAN_SSSE3_DISPATCH +#undef JSON_SCAN_SSSE3_TARGET +#undef JSON_SCAN_VECTOR +#undef JSON_SCAN_VECTOR_UTF8 #ifndef JSON_TEST_KEEP_MACROS #undef JSON_CATCH diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 28bf2e7fa..6c6099248 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -373,6 +373,25 @@ json_test_add_test_for(src/unit-diagnostic-positions.cpp MAIN test_main CXX_STANDARDS ${test_cxx_standards} ${test_force} ) +# the lexer again with the portable string scanning instead of SSE2/NEON, and on +# x86-64 with SSSE3 compiled in (no check of the CPU) +json_test_set_test_options(test-class_lexer_no_simd + COMPILE_DEFINITIONS JSON_NO_SIMD +) +json_test_add_test_for(src/unit-class_lexer.cpp + NAME test-class_lexer_no_simd + MAIN test_main CXX_STANDARDS ${test_cxx_standards} ${test_force} +) +if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64|amd64)$" AND NOT MSVC) + json_test_set_test_options(test-class_lexer_ssse3 + COMPILE_OPTIONS -mssse3 + ) + json_test_add_test_for(src/unit-class_lexer.cpp + NAME test-class_lexer_ssse3 + MAIN test_main CXX_STANDARDS ${test_cxx_standards} ${test_force} + ) +endif() + # test reading from a std::istream again without scanning the get area in bulk json_test_set_test_options(test-stream-bulk-scan_disabled COMPILE_DEFINITIONS JSON_DISABLE_STREAM_BULK_SCAN=1 diff --git a/tests/src/unit-class_lexer.cpp b/tests/src/unit-class_lexer.cpp index 91c835db3..1e9f19a41 100644 --- a/tests/src/unit-class_lexer.cpp +++ b/tests/src/unit-class_lexer.cpp @@ -716,6 +716,73 @@ TEST_CASE("lexer string fast path") CAPTURE(mismatches) CHECK(mismatches.empty()); } + + SECTION("special bytes at every offset of the vector blocks") + { + // With SSE2 or NEON, runs are scanned 16 bytes at a time (after one + // SWAR word, or four with NEON), and runs of non-ASCII text are + // checked 16 bytes at a time as well. Place every kind of byte that + // ends a run at each offset across several blocks, after ASCII and + // after CJK text, so sequences also straddle the block boundaries. + // The reference is a std::list, which has no contiguous storage and + // is therefore read one character at a time. + const auto byte_path = [](const std::string & doc) -> std::string + { + const std::list chars(doc.begin(), doc.end()); + JSON_TRY + { + return json::parse(chars.begin(), chars.end()).dump(); + } + JSON_CATCH (const json::exception & e) + { + return e.what(); + } + }; + const std::vector specials = + { + "\"", "\\", "\\u00e9", bytes({0x01}), bytes({0x1F}), bytes({0x7F}), + bytes({0xC3, 0xA9}), bytes({0xE4, 0xB8, 0xAD}), bytes({0xF0, 0x9F, 0x98, 0x80}), + bytes({0xFF}), bytes({0xC3}), bytes({0xE4, 0xB8}), bytes({0xED, 0xA0, 0x80}), bytes({0xC0, 0x80}) + }; + const std::string cjk = bytes({0xE4, 0xB8, 0xAD}); // U+4E2D + + std::vector mismatches; + for (std::size_t offset = 0; offset <= 80; ++offset) + { + std::string text; + while (text.size() < offset) + { + text += cjk; + } + for (const auto& prefix : + { + std::string(offset, 'a'), text + }) + { + for (const auto& special : specials) + { + // the closing quote right behind, or in a later block + for (const int tail : + { + 0, 1, 3, 20 + }) + { + std::string doc = "[\""; + doc += prefix; + doc += special; + doc.append(static_cast(tail), 'b'); + doc += "\"]"; + if (outcome(doc, false) != byte_path(doc)) + { + mismatches.push_back(doc); + } + } + } + } + } + CAPTURE(mismatches) + CHECK(mismatches.empty()); + } #endif // json::accept() never throws, so the ranges stay covered without exceptions