diff --git a/docs/mkdocs/docs/api/macros/index.md b/docs/mkdocs/docs/api/macros/index.md index 507c04932..e818f032a 100644 --- a/docs/mkdocs/docs/api/macros/index.md +++ b/docs/mkdocs/docs/api/macros/index.md @@ -24,6 +24,7 @@ header. See also the [macro overview page](../../features/macros.md). - [**JSON_NO_IO**](json_no_io.md) - switch off functions relying on certain C++ I/O headers - [**JSON_SKIP_UNSUPPORTED_COMPILER_CHECK**](json_skip_unsupported_compiler_check.md) - do not warn about unsupported compilers - [**JSON_USE_GLOBAL_UDLS**](json_use_global_udls.md) - place user-defined string literals (UDLs) into the global namespace +- [**JSON_USE_SIMDUTF**](json_use_simdutf.md) - use the simdutf library to accelerate UTF-8 validation ## Library version diff --git a/docs/mkdocs/docs/api/macros/json_use_simdutf.md b/docs/mkdocs/docs/api/macros/json_use_simdutf.md new file mode 100644 index 000000000..097ce795b --- /dev/null +++ b/docs/mkdocs/docs/api/macros/json_use_simdutf.md @@ -0,0 +1,52 @@ +# JSON_USE_SIMDUTF + +```cpp +#define JSON_USE_SIMDUTF +``` + +When defined, the parser validates the UTF-8 content of JSON strings that come from a **contiguous byte input** +(`std::string`, `std::vector`/``, string literals, `const char*` ranges, …) using the +[simdutf](https://github.com/simdutf/simdutf) library instead of the built-in scalar validator. On text with many +non-ASCII characters (e.g. CJK or emoji) this can validate several times faster. + +This is an **opt-in external dependency**. The library itself remains header-only and its behavior is unchanged: the +same input is accepted or rejected either way, and every parse error is reported at the same position with the same +message (simdutf is only used to fast-path *valid* runs; anything it flags falls back to the scalar path so the exact +diagnostic is preserved). Streaming inputs (files, `std::istream`, wide strings, user-defined adapters) always use the +scalar path. + +When `JSON_USE_SIMDUTF` is defined you must make the `simdutf.h` header available on the include path and link the +simdutf library. When it is not defined, no simdutf header is included and there is no dependency. + +## Default definition + +By default, `#!cpp JSON_USE_SIMDUTF` is not defined and the portable C++11 scalar validator is used. + +```cpp +#undef JSON_USE_SIMDUTF +``` + +## Examples + +??? example + + The code below enables the simdutf backend for UTF-8 validation. + + ```cpp + #define JSON_USE_SIMDUTF 1 + #include + #include + + ... + ``` + + The project must also link against simdutf, e.g. with CMake: + + ```cmake + target_compile_definitions(your_target PRIVATE JSON_USE_SIMDUTF) + target_link_libraries(your_target PRIVATE simdutf::simdutf) + ``` + +## Version history + +- Added in version 3.12.1. diff --git a/include/nlohmann/detail/input/lexer.hpp b/include/nlohmann/detail/input/lexer.hpp index ad1835920..e042d81eb 100644 --- a/include/nlohmann/detail/input/lexer.hpp +++ b/include/nlohmann/detail/input/lexer.hpp @@ -21,6 +21,15 @@ #include // move #include // vector +#if defined(JSON_USE_SIMDUTF) + // Optional SIMD backend for bulk UTF-8 validation. This is an opt-in + // external dependency: nlohmann/json itself stays header-only and the C++11 + // scalar validator below is always available; defining JSON_USE_SIMDUTF + // additionally requires the simdutf headers on the include path and linking + // the simdutf library. See scan_string_bulk(). + #include +#endif + #include #include #include @@ -414,6 +423,90 @@ class lexer : public lexer_base return 0; // invalid, incomplete, or must be diagnosed by the byte path } + // Scalar (C++11) computation of the bulk run length: the number of leading + // bytes in [data, data+n) that are ordinary ASCII or complete well-formed + // UTF-8 sequences, stopping before the first byte that needs individual + // handling (the closing quote, an escape, a control character, or an + // ill-formed/truncated sequence). ASCII is skipped 8 bytes at a time. + static std::size_t scalar_string_bulk_run(const unsigned char* data, std::size_t n) noexcept + { + std::size_t pos = 0; + while (pos < n) + { + pos += find_string_special(data + pos, n - pos); + if (pos >= n || data[pos] < 0x80u) + { + break; // end of buffer, or a quote/escape/control byte + } + const std::size_t seq = validate_one_utf8(data + pos, n - pos); + if (seq == 0) + { + break; // ill-formed or truncated: let the byte path diagnose it + } + pos += seq; + } + return pos; + } + +#if defined(JSON_USE_SIMDUTF) + // Index of the first quote/escape/control byte in [data, data+n) (non-ASCII + // bytes are *not* stops here - the whole run is handed to simdutf), or n. + static std::size_t find_string_delimiter(const unsigned char* data, std::size_t n) noexcept + { + constexpr std::uint64_t ones = 0x0101010101010101ull; + constexpr std::uint64_t high = 0x8080808080808080ull; + std::size_t i = 0; + for (; i + 8 <= n; i += 8) + { + std::uint64_t v = 0; + std::memcpy(&v, data + i, sizeof(v)); + const std::uint64_t q = v ^ 0x2222222222222222ull; + const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; + const std::uint64_t hit = ((q - ones) & ~q & high) + | ((b - ones) & ~b & high) + | ((v - 0x2020202020202020ull) & ~v & high); + if (hit != 0) + { + for (std::size_t j = 0; j < 8; ++j) + { + const unsigned char c = data[i + j]; + if (c == '\"' || c == '\\' || c < 0x20u) + { + return i + j; + } + } + } + } + for (; i < n; ++i) + { + const unsigned char c = data[i]; + if (c == '\"' || c == '\\' || c < 0x20u) + { + return i; + } + } + return n; + } +#endif + + // Backend-dispatched bulk run length. With JSON_USE_SIMDUTF the run up to the + // next delimiter is validated in one shot by simdutf; on the rare failure the + // scalar helper recomputes the exact valid prefix so the byte path still + // produces the precise diagnostic. Without it, the pure scalar path is used. + static std::size_t string_bulk_run(const unsigned char* data, std::size_t n) noexcept + { +#if defined(JSON_USE_SIMDUTF) + const std::size_t run = find_string_delimiter(data, n); + if (run != 0 && simdutf::validate_utf8(reinterpret_cast(data), run)) + { + return run; + } + return scalar_string_bulk_run(data, n); +#else + return scalar_string_bulk_run(data, n); +#endif + } + /// contiguous input: bulk-append the run of ordinary characters and complete /// well-formed UTF-8 sequences starting at the current read position, leaving /// the first byte that needs individual handling (the closing quote, an @@ -432,31 +525,7 @@ class lexer : public lexer_base } const auto* const data = reinterpret_cast(ia.bulk_data()); - std::size_t pos = 0; - while (pos < remaining) - { - // bulk-skip ordinary ASCII, stopping at the next special byte - pos += find_string_special(data + pos, remaining - pos); - if (pos >= remaining) - { - break; - } - const unsigned char c = data[pos]; - if (c < 0x80u) - { - // closing quote, escape, or control character: let get() handle it - break; - } - // a non-ASCII lead byte: fold a well-formed sequence into the run, - // otherwise stop and let the byte path produce the exact diagnostic - const std::size_t seq = validate_one_utf8(data + pos, remaining - pos); - if (seq == 0) - { - break; - } - pos += seq; - } - + const std::size_t pos = string_bulk_run(data, remaining); if (pos == 0) { return; diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index d3241cf62..a45625ef5 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -7780,6 +7780,15 @@ NLOHMANN_JSON_NAMESPACE_END #include // move #include // vector +#if defined(JSON_USE_SIMDUTF) + // Optional SIMD backend for bulk UTF-8 validation. This is an opt-in + // external dependency: nlohmann/json itself stays header-only and the C++11 + // scalar validator below is always available; defining JSON_USE_SIMDUTF + // additionally requires the simdutf headers on the include path and linking + // the simdutf library. See scan_string_bulk(). + #include +#endif + // #include // #include @@ -8177,6 +8186,90 @@ class lexer : public lexer_base return 0; // invalid, incomplete, or must be diagnosed by the byte path } + // Scalar (C++11) computation of the bulk run length: the number of leading + // bytes in [data, data+n) that are ordinary ASCII or complete well-formed + // UTF-8 sequences, stopping before the first byte that needs individual + // handling (the closing quote, an escape, a control character, or an + // ill-formed/truncated sequence). ASCII is skipped 8 bytes at a time. + static std::size_t scalar_string_bulk_run(const unsigned char* data, std::size_t n) noexcept + { + std::size_t pos = 0; + while (pos < n) + { + pos += find_string_special(data + pos, n - pos); + if (pos >= n || data[pos] < 0x80u) + { + break; // end of buffer, or a quote/escape/control byte + } + const std::size_t seq = validate_one_utf8(data + pos, n - pos); + if (seq == 0) + { + break; // ill-formed or truncated: let the byte path diagnose it + } + pos += seq; + } + return pos; + } + +#if defined(JSON_USE_SIMDUTF) + // Index of the first quote/escape/control byte in [data, data+n) (non-ASCII + // bytes are *not* stops here - the whole run is handed to simdutf), or n. + static std::size_t find_string_delimiter(const unsigned char* data, std::size_t n) noexcept + { + constexpr std::uint64_t ones = 0x0101010101010101ull; + constexpr std::uint64_t high = 0x8080808080808080ull; + std::size_t i = 0; + for (; i + 8 <= n; i += 8) + { + std::uint64_t v = 0; + std::memcpy(&v, data + i, sizeof(v)); + const std::uint64_t q = v ^ 0x2222222222222222ull; + const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; + const std::uint64_t hit = ((q - ones) & ~q & high) + | ((b - ones) & ~b & high) + | ((v - 0x2020202020202020ull) & ~v & high); + if (hit != 0) + { + for (std::size_t j = 0; j < 8; ++j) + { + const unsigned char c = data[i + j]; + if (c == '\"' || c == '\\' || c < 0x20u) + { + return i + j; + } + } + } + } + for (; i < n; ++i) + { + const unsigned char c = data[i]; + if (c == '\"' || c == '\\' || c < 0x20u) + { + return i; + } + } + return n; + } +#endif + + // Backend-dispatched bulk run length. With JSON_USE_SIMDUTF the run up to the + // next delimiter is validated in one shot by simdutf; on the rare failure the + // scalar helper recomputes the exact valid prefix so the byte path still + // produces the precise diagnostic. Without it, the pure scalar path is used. + static std::size_t string_bulk_run(const unsigned char* data, std::size_t n) noexcept + { +#if defined(JSON_USE_SIMDUTF) + const std::size_t run = find_string_delimiter(data, n); + if (run != 0 && simdutf::validate_utf8(reinterpret_cast(data), run)) + { + return run; + } + return scalar_string_bulk_run(data, n); +#else + return scalar_string_bulk_run(data, n); +#endif + } + /// contiguous input: bulk-append the run of ordinary characters and complete /// well-formed UTF-8 sequences starting at the current read position, leaving /// the first byte that needs individual handling (the closing quote, an @@ -8195,31 +8288,7 @@ class lexer : public lexer_base } const auto* const data = reinterpret_cast(ia.bulk_data()); - std::size_t pos = 0; - while (pos < remaining) - { - // bulk-skip ordinary ASCII, stopping at the next special byte - pos += find_string_special(data + pos, remaining - pos); - if (pos >= remaining) - { - break; - } - const unsigned char c = data[pos]; - if (c < 0x80u) - { - // closing quote, escape, or control character: let get() handle it - break; - } - // a non-ASCII lead byte: fold a well-formed sequence into the run, - // otherwise stop and let the byte path produce the exact diagnostic - const std::size_t seq = validate_one_utf8(data + pos, remaining - pos); - if (seq == 0) - { - break; - } - pos += seq; - } - + const std::size_t pos = string_bulk_run(data, remaining); if (pos == 0) { return;