diff --git a/include/nlohmann/detail/input/simd_scan.hpp b/include/nlohmann/detail/input/simd_scan.hpp index 6c34421fe..e667c8366 100644 --- a/include/nlohmann/detail/input/simd_scan.hpp +++ b/include/nlohmann/detail/input/simd_scan.hpp @@ -92,8 +92,12 @@ namespace detail character, or not ASCII, 16 bytes per step Stops at such a byte, or where fewer than 16 bytes are left (the caller tells -the two apart). A signed compare with 0x20 finds control characters and -non-ASCII bytes at once. +the two apart). One signed compare finds the quote, the control characters and +the non-ASCII bytes at once: flipping bit 1 maps 0x00-0x1F onto itself and the +quote (0x22) onto 0x20, keeps 0x80-0xFF negative, and maps every other byte to +33 or more, so these are exactly the bytes with (int8_t)(c ^ 2) < 33 (see +D. Lemire, "Detect control characters, quotes and backslashes efficiently +using SWAR", 2025). The backslash needs its own compare. */ JSON_HEDLEY_ALWAYS_INLINE const unsigned char* vector_plain_run(const unsigned char* p, const unsigned char* e) noexcept { @@ -101,8 +105,8 @@ JSON_HEDLEY_ALWAYS_INLINE const unsigned char* vector_plain_run(const unsigned c { #if JSON_SCAN_NEON const uint8x16_t in = vld1q_u8(p); - const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('"')), vceqq_u8(in, vdupq_n_u8('\\'))), - vcltq_s8(vreinterpretq_s8_u8(in), vdupq_n_s8(0x20))); + const uint8x16_t special = vorrq_u8(vceqq_u8(in, vdupq_n_u8('\\')), + vcltq_s8(vreinterpretq_s8_u8(veorq_u8(in, vdupq_n_u8(2))), vdupq_n_s8(33))); // one nibble per byte (the usual NEON replacement of x86's movemask, see // D. Kutenin, "Porting x86 vector bitmask optimizations to Arm NEON", 2022) const std::uint64_t bits = vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(special), 4)), 0); @@ -112,8 +116,8 @@ JSON_HEDLEY_ALWAYS_INLINE const unsigned char* vector_plain_run(const unsigned c } #else const __m128i in = _mm_loadu_si128(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) - const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), - _mm_cmplt_epi8(in, _mm_set1_epi8(0x20))); + const __m128i special = _mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('\\')), + _mm_cmplt_epi8(_mm_xor_si128(in, _mm_set1_epi8(2)), _mm_set1_epi8(33))); const auto bits = static_cast(static_cast(_mm_movemask_epi8(special))); if (bits != 0) { @@ -250,7 +254,8 @@ JSON_SCAN_SSSE3_TARGET JSON_HEDLEY_NEVER_INLINE inline bool vector_utf8_run(cons while (e - block >= 16) { const uint8x16_t in = vld1q_u8(block); - const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('"')), vceqq_u8(in, vdupq_n_u8('\\'))), vcltq_u8(in, vdupq_n_u8(0x20))); + // the quote and the control characters: (c ^ 2) < 33 (see vector_plain_run()) + const uint8x16_t special = vorrq_u8(vceqq_u8(in, vdupq_n_u8('\\')), vcltq_u8(veorq_u8(in, vdupq_n_u8(2)), vdupq_n_u8(33))); const uint8x16_t prev1 = vextq_u8(prev, in, 15); const uint8x16_t sc = vandq_u8(vandq_u8(vqtbl1q_u8(t1h, vshrq_n_u8(prev1, 4)), vqtbl1q_u8(t1l, vandq_u8(prev1, vdupq_n_u8(0x0F)))), vqtbl1q_u8(t2h, vshrq_n_u8(in, 4))); const uint8x16_t must23 = vorrq_u8(vqsubq_u8(vextq_u8(prev, in, 14), vdupq_n_u8(0xE0 - 0x80)), vqsubq_u8(vextq_u8(prev, in, 13), vdupq_n_u8(0xF0 - 0x80))); @@ -290,8 +295,10 @@ JSON_SCAN_SSSE3_TARGET JSON_HEDLEY_NEVER_INLINE inline bool vector_utf8_run(cons while (e - block >= 16) { const __m128i in = _mm_loadu_si128(static_cast(static_cast(block))); // NOLINT(bugprone-casting-through-void) - const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), - _mm_cmpeq_epi8(_mm_subs_epu8(in, _mm_set1_epi8(0x1F)), zero)); // in < 0x20 + // the quote and the control characters: (c ^ 2) < 33, that is, + // (c ^ 2) - 32 saturates to 0 (see vector_plain_run()) + const __m128i special = _mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('\\')), + _mm_cmpeq_epi8(_mm_subs_epu8(_mm_xor_si128(in, _mm_set1_epi8(2)), _mm_set1_epi8(32)), zero)); const __m128i prev1 = _mm_alignr_epi8(in, prev, 15); const __m128i sc = _mm_and_si128(_mm_and_si128(_mm_shuffle_epi8(t1h, _mm_and_si128(_mm_srli_epi16(prev1, 4), nibble)), _mm_shuffle_epi8(t1l, _mm_and_si128(prev1, nibble))), diff --git a/include/nlohmann/detail/output/serializer.hpp b/include/nlohmann/detail/output/serializer.hpp index d3cf93d3b..a2ba87205 100644 --- a/include/nlohmann/detail/output/serializer.hpp +++ b/include/nlohmann/detail/output/serializer.hpp @@ -834,8 +834,10 @@ class serializer const uint8x8_t lo = vld1_u8(reinterpret_cast(p)); const uint8x8_t hi = vld1_u8(reinterpret_cast(p + n - 8)); const uint8x16_t in = vcombine_u8(lo, hi); - const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('"')), vceqq_u8(in, vdupq_n_u8('\\'))), - vorrq_u8(vcltq_s8(vreinterpretq_s8_u8(in), vdupq_n_s8(0x20)), vceqq_u8(in, vdupq_n_u8(0x7F)))); + // (int8_t)(c ^ 2) < 33: the quote, the control characters and the + // non-ASCII bytes (see vector_plain_run()) + const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('\\')), vceqq_u8(in, vdupq_n_u8(0x7F))), + vcltq_s8(vreinterpretq_s8_u8(veorq_u8(in, vdupq_n_u8(2))), vdupq_n_s8(33))); if (vmaxvq_u8(special) != 0) { return false; @@ -846,8 +848,8 @@ class serializer const __m128i lo = _mm_loadl_epi64(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) const __m128i hi = _mm_loadl_epi64(static_cast(static_cast(p + n - 8))); // NOLINT(bugprone-casting-through-void) const __m128i in = _mm_unpacklo_epi64(lo, hi); - const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), - _mm_or_si128(_mm_cmplt_epi8(in, _mm_set1_epi8(0x20)), _mm_cmpeq_epi8(in, _mm_set1_epi8(0x7F)))); + const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('\\')), _mm_cmpeq_epi8(in, _mm_set1_epi8(0x7F))), + _mm_cmplt_epi8(_mm_xor_si128(in, _mm_set1_epi8(2)), _mm_set1_epi8(33))); if (_mm_movemask_epi8(special) != 0) { return false; @@ -866,10 +868,13 @@ class serializer #if JSON_SCAN_NEON const uint8x16_t a = vld1q_u8(reinterpret_cast(p)); const uint8x16_t b = vld1q_u8(reinterpret_cast(p + n - 16)); - const uint8x16_t special = vorrq_u8(vorrq_u8(vorrq_u8(vceqq_u8(a, vdupq_n_u8('"')), vceqq_u8(a, vdupq_n_u8('\\'))), - vorrq_u8(vceqq_u8(b, vdupq_n_u8('"')), vceqq_u8(b, vdupq_n_u8('\\')))), - vorrq_u8(vorrq_u8(vcltq_s8(vreinterpretq_s8_u8(a), vdupq_n_s8(0x20)), vcltq_s8(vreinterpretq_s8_u8(b), vdupq_n_s8(0x20))), - vorrq_u8(vceqq_u8(a, vdupq_n_u8(0x7F)), vceqq_u8(b, vdupq_n_u8(0x7F))))); + // (int8_t)(c ^ 2) < 33: the quote, the control characters and the + // non-ASCII bytes (see vector_plain_run()) + const uint8x16_t flip = vdupq_n_u8(2); + const int8x16_t limit = vdupq_n_s8(33); + const uint8x16_t special = vorrq_u8(vorrq_u8(vorrq_u8(vceqq_u8(a, vdupq_n_u8('\\')), vceqq_u8(b, vdupq_n_u8('\\'))), + vorrq_u8(vceqq_u8(a, vdupq_n_u8(0x7F)), vceqq_u8(b, vdupq_n_u8(0x7F)))), + vorrq_u8(vcltq_s8(vreinterpretq_s8_u8(veorq_u8(a, flip)), limit), vcltq_s8(vreinterpretq_s8_u8(veorq_u8(b, flip)), limit))); if (vmaxvq_u8(special) != 0) { return false; @@ -879,14 +884,15 @@ class serializer #else const __m128i a = _mm_loadu_si128(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) const __m128i b = _mm_loadu_si128(static_cast(static_cast(p + n - 16))); // NOLINT(bugprone-casting-through-void) - const __m128i quote = _mm_set1_epi8('"'); + // (int8_t)(c ^ 2) < 33: the quote, the control characters and the + // non-ASCII bytes (see vector_plain_run()) const __m128i backslash = _mm_set1_epi8('\\'); - const __m128i space = _mm_set1_epi8(0x20); const __m128i del = _mm_set1_epi8(0x7F); - const __m128i special = _mm_or_si128(_mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(a, quote), _mm_cmpeq_epi8(a, backslash)), - _mm_or_si128(_mm_cmpeq_epi8(b, quote), _mm_cmpeq_epi8(b, backslash))), - _mm_or_si128(_mm_or_si128(_mm_cmplt_epi8(a, space), _mm_cmplt_epi8(b, space)), - _mm_or_si128(_mm_cmpeq_epi8(a, del), _mm_cmpeq_epi8(b, del)))); + const __m128i flip = _mm_set1_epi8(2); + const __m128i limit = _mm_set1_epi8(33); + const __m128i special = _mm_or_si128(_mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(a, backslash), _mm_cmpeq_epi8(b, backslash)), + _mm_or_si128(_mm_cmpeq_epi8(a, del), _mm_cmpeq_epi8(b, del))), + _mm_or_si128(_mm_cmplt_epi8(_mm_xor_si128(a, flip), limit), _mm_cmplt_epi8(_mm_xor_si128(b, flip), limit))); if (_mm_movemask_epi8(special) != 0) { return false; diff --git a/include/nlohmann/detail/view/serializer.hpp b/include/nlohmann/detail/view/serializer.hpp index deccec8e4..fb40b1323 100644 --- a/include/nlohmann/detail/view/serializer.hpp +++ b/include/nlohmann/detail/view/serializer.hpp @@ -152,9 +152,12 @@ inline std::size_t plain_output_run(const unsigned char* s, std::size_t n) noexc for (; i + 16 <= n; i += 16) { const __m128i in = _mm_loadu_si128(static_cast(static_cast(s + i))); // NOLINT(bugprone-casting-through-void) - // (a byte is below 0x20 if it is its own minimum with 0x1F) - const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), - _mm_cmpeq_epi8(_mm_min_epu8(in, _mm_set1_epi8(0x1F)), in)); + // the quote and the control characters are the bytes with + // (c ^ 2) <= 32, that is, those that are their own minimum with 32 + // (see vector_plain_run()) + const __m128i flipped = _mm_xor_si128(in, _mm_set1_epi8(2)); + const __m128i special = _mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('\\')), + _mm_cmpeq_epi8(_mm_min_epu8(flipped, _mm_set1_epi8(32)), flipped)); const auto mask = static_cast(static_cast(_mm_movemask_epi8(special))); if (mask != 0) { @@ -217,20 +220,21 @@ inline bool is_ascii_without_delete(const char* s, std::size_t n) noexcept #if NLOHMANN_VIEW_VECTOR /// the bytes of the 16 at p that ensure_ascii cannot copy (a quote, a -/// backslash, a control character, DEL, or not ASCII; a signed compare with -/// 0x20 finds control characters and non-ASCII bytes at once), one bit per -/// byte (x86) or one nibble per byte (NEON) +/// backslash, a control character, DEL, or not ASCII; the signed compare +/// (int8_t)(c ^ 2) < 33 finds the quote, the control characters and the +/// non-ASCII bytes at once, see vector_plain_run()), one bit per byte (x86) or +/// one nibble per byte (NEON) NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t ascii_stop_mask(const unsigned char* p) noexcept { #if NLOHMANN_VIEW_NEON const uint8x16_t in = vld1q_u8(p); - const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('"')), vceqq_u8(in, vdupq_n_u8('\\'))), - vorrq_u8(vceqq_u8(in, vdupq_n_u8(0x7F)), vcltq_s8(vreinterpretq_s8_u8(in), vdupq_n_s8(0x20)))); + const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('\\')), vceqq_u8(in, vdupq_n_u8(0x7F))), + vcltq_s8(vreinterpretq_s8_u8(veorq_u8(in, vdupq_n_u8(2))), vdupq_n_s8(33))); return vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(special), 4)), 0); #else const __m128i in = _mm_loadu_si128(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) - const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), - _mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8(0x7F)), _mm_cmplt_epi8(in, _mm_set1_epi8(0x20)))); + const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('\\')), _mm_cmpeq_epi8(in, _mm_set1_epi8(0x7F))), + _mm_cmplt_epi8(_mm_xor_si128(in, _mm_set1_epi8(2)), _mm_set1_epi8(33))); return static_cast(static_cast(_mm_movemask_epi8(special))); #endif } diff --git a/include/nlohmann/detail/view/simd.hpp b/include/nlohmann/detail/view/simd.hpp index f4745f6b8..b1fc8ba74 100644 --- a/include/nlohmann/detail/view/simd.hpp +++ b/include/nlohmann/detail/view/simd.hpp @@ -91,8 +91,10 @@ namespace view character, or not ASCII, 16 bytes per step Stops at such a byte, or where fewer than 16 bytes are left (the caller tells -the two apart). A signed compare with 0x20 finds control characters and -non-ASCII bytes at once. +the two apart). One signed compare finds the quote, the control characters and +the non-ASCII bytes at once: these are exactly the bytes with +(int8_t)(c ^ 2) < 33 (see detail::vector_plain_run()). The backslash needs its +own compare. */ NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* vector_plain_run(const unsigned char* p, const unsigned char* e) noexcept { @@ -100,8 +102,8 @@ NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* vector_plain_run(const unsigned { #if NLOHMANN_VIEW_NEON const uint8x16_t in = vld1q_u8(p); - const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('"')), vceqq_u8(in, vdupq_n_u8('\\'))), - vcltq_s8(vreinterpretq_s8_u8(in), vdupq_n_s8(0x20))); + const uint8x16_t special = vorrq_u8(vceqq_u8(in, vdupq_n_u8('\\')), + vcltq_s8(vreinterpretq_s8_u8(veorq_u8(in, vdupq_n_u8(2))), vdupq_n_s8(33))); // one nibble per byte (the usual NEON replacement of x86's movemask, see // D. Kutenin, "Porting x86 vector bitmask optimizations to Arm NEON", 2022) const std::uint64_t bits = vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(special), 4)), 0); @@ -111,8 +113,8 @@ NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* vector_plain_run(const unsigned } #else const __m128i in = _mm_loadu_si128(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) - const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), - _mm_cmplt_epi8(in, _mm_set1_epi8(0x20))); + const __m128i special = _mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('\\')), + _mm_cmplt_epi8(_mm_xor_si128(in, _mm_set1_epi8(2)), _mm_set1_epi8(33))); const auto bits = static_cast(static_cast(_mm_movemask_epi8(special))); if (bits != 0) { @@ -138,15 +140,17 @@ NLOHMANN_VIEW_ALWAYS_INLINE unsigned copy_plain_run16(const unsigned char* p, ch #if NLOHMANN_VIEW_NEON const uint8x16_t in = vld1q_u8(p); vst1q_u8(reinterpret_cast(w), in); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) - const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('"')), vceqq_u8(in, vdupq_n_u8('\\'))), - vcltq_s8(vreinterpretq_s8_u8(in), vdupq_n_s8(0x20))); + // (int8_t)(c ^ 2) < 33: the quote, the control characters and the + // non-ASCII bytes (see vector_plain_run()) + const uint8x16_t special = vorrq_u8(vceqq_u8(in, vdupq_n_u8('\\')), + vcltq_s8(vreinterpretq_s8_u8(veorq_u8(in, vdupq_n_u8(2))), vdupq_n_s8(33))); const std::uint64_t bits = vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(special), 4)), 0); return bits != 0 ? static_cast(count_trailing_zeros(bits) >> 2u) : 16u; #else const __m128i in = _mm_loadu_si128(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) _mm_storeu_si128(static_cast<__m128i*>(static_cast(w)), in); // NOLINT(bugprone-casting-through-void) - const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), - _mm_cmplt_epi8(in, _mm_set1_epi8(0x20))); + const __m128i special = _mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('\\')), + _mm_cmplt_epi8(_mm_xor_si128(in, _mm_set1_epi8(2)), _mm_set1_epi8(33))); const auto bits = static_cast(static_cast(_mm_movemask_epi8(special))); return bits != 0 ? static_cast(count_trailing_zeros(bits)) : 16u; #endif @@ -306,7 +310,8 @@ NLOHMANN_VIEW_SSSE3_TARGET NLOHMANN_VIEW_NOINLINE NLOHMANN_VIEW_NOCLONE inline c while (e - block >= 16) { const uint8x16_t in = vld1q_u8(block); - const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('"')), vceqq_u8(in, vdupq_n_u8('\\'))), vcltq_u8(in, vdupq_n_u8(0x20))); + // the quote and the control characters: (c ^ 2) < 33 (see vector_plain_run()) + const uint8x16_t special = vorrq_u8(vceqq_u8(in, vdupq_n_u8('\\')), vcltq_u8(veorq_u8(in, vdupq_n_u8(2)), vdupq_n_u8(33))); const uint8x16_t prev1 = vextq_u8(prev, in, 15); const uint8x16_t sc = vandq_u8(vandq_u8(vqtbl1q_u8(t1h, vshrq_n_u8(prev1, 4)), vqtbl1q_u8(t1l, vandq_u8(prev1, vdupq_n_u8(0x0F)))), vqtbl1q_u8(t2h, vshrq_n_u8(in, 4))); const uint8x16_t must23 = vorrq_u8(vqsubq_u8(vextq_u8(prev, in, 14), vdupq_n_u8(0xE0 - 0x80)), vqsubq_u8(vextq_u8(prev, in, 13), vdupq_n_u8(0xF0 - 0x80))); @@ -345,8 +350,10 @@ NLOHMANN_VIEW_SSSE3_TARGET NLOHMANN_VIEW_NOINLINE NLOHMANN_VIEW_NOCLONE inline c while (e - block >= 16) { const __m128i in = _mm_loadu_si128(static_cast(static_cast(block))); // NOLINT(bugprone-casting-through-void) - const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), - _mm_cmpeq_epi8(_mm_subs_epu8(in, _mm_set1_epi8(0x1F)), zero)); // in < 0x20 + // the quote and the control characters: (c ^ 2) < 33, that is, + // (c ^ 2) - 32 saturates to 0 (see vector_plain_run()) + const __m128i special = _mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('\\')), + _mm_cmpeq_epi8(_mm_subs_epu8(_mm_xor_si128(in, _mm_set1_epi8(2)), _mm_set1_epi8(32)), zero)); const __m128i prev1 = _mm_alignr_epi8(in, prev, 15); const __m128i sc = _mm_and_si128(_mm_and_si128(_mm_shuffle_epi8(t1h, _mm_and_si128(_mm_srli_epi16(prev1, 4), nibble)), _mm_shuffle_epi8(t1l, _mm_and_si128(prev1, nibble))), diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 3725c084a..6d3be4e90 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -12186,8 +12186,12 @@ namespace detail character, or not ASCII, 16 bytes per step Stops at such a byte, or where fewer than 16 bytes are left (the caller tells -the two apart). A signed compare with 0x20 finds control characters and -non-ASCII bytes at once. +the two apart). One signed compare finds the quote, the control characters and +the non-ASCII bytes at once: flipping bit 1 maps 0x00-0x1F onto itself and the +quote (0x22) onto 0x20, keeps 0x80-0xFF negative, and maps every other byte to +33 or more, so these are exactly the bytes with (int8_t)(c ^ 2) < 33 (see +D. Lemire, "Detect control characters, quotes and backslashes efficiently +using SWAR", 2025). The backslash needs its own compare. */ JSON_HEDLEY_ALWAYS_INLINE const unsigned char* vector_plain_run(const unsigned char* p, const unsigned char* e) noexcept { @@ -12195,8 +12199,8 @@ JSON_HEDLEY_ALWAYS_INLINE const unsigned char* vector_plain_run(const unsigned c { #if JSON_SCAN_NEON const uint8x16_t in = vld1q_u8(p); - const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('"')), vceqq_u8(in, vdupq_n_u8('\\'))), - vcltq_s8(vreinterpretq_s8_u8(in), vdupq_n_s8(0x20))); + const uint8x16_t special = vorrq_u8(vceqq_u8(in, vdupq_n_u8('\\')), + vcltq_s8(vreinterpretq_s8_u8(veorq_u8(in, vdupq_n_u8(2))), vdupq_n_s8(33))); // one nibble per byte (the usual NEON replacement of x86's movemask, see // D. Kutenin, "Porting x86 vector bitmask optimizations to Arm NEON", 2022) const std::uint64_t bits = vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(special), 4)), 0); @@ -12206,8 +12210,8 @@ JSON_HEDLEY_ALWAYS_INLINE const unsigned char* vector_plain_run(const unsigned c } #else const __m128i in = _mm_loadu_si128(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) - const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), - _mm_cmplt_epi8(in, _mm_set1_epi8(0x20))); + const __m128i special = _mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('\\')), + _mm_cmplt_epi8(_mm_xor_si128(in, _mm_set1_epi8(2)), _mm_set1_epi8(33))); const auto bits = static_cast(static_cast(_mm_movemask_epi8(special))); if (bits != 0) { @@ -12344,7 +12348,8 @@ JSON_SCAN_SSSE3_TARGET JSON_HEDLEY_NEVER_INLINE inline bool vector_utf8_run(cons while (e - block >= 16) { const uint8x16_t in = vld1q_u8(block); - const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('"')), vceqq_u8(in, vdupq_n_u8('\\'))), vcltq_u8(in, vdupq_n_u8(0x20))); + // the quote and the control characters: (c ^ 2) < 33 (see vector_plain_run()) + const uint8x16_t special = vorrq_u8(vceqq_u8(in, vdupq_n_u8('\\')), vcltq_u8(veorq_u8(in, vdupq_n_u8(2)), vdupq_n_u8(33))); const uint8x16_t prev1 = vextq_u8(prev, in, 15); const uint8x16_t sc = vandq_u8(vandq_u8(vqtbl1q_u8(t1h, vshrq_n_u8(prev1, 4)), vqtbl1q_u8(t1l, vandq_u8(prev1, vdupq_n_u8(0x0F)))), vqtbl1q_u8(t2h, vshrq_n_u8(in, 4))); const uint8x16_t must23 = vorrq_u8(vqsubq_u8(vextq_u8(prev, in, 14), vdupq_n_u8(0xE0 - 0x80)), vqsubq_u8(vextq_u8(prev, in, 13), vdupq_n_u8(0xF0 - 0x80))); @@ -12384,8 +12389,10 @@ JSON_SCAN_SSSE3_TARGET JSON_HEDLEY_NEVER_INLINE inline bool vector_utf8_run(cons while (e - block >= 16) { const __m128i in = _mm_loadu_si128(static_cast(static_cast(block))); // NOLINT(bugprone-casting-through-void) - const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), - _mm_cmpeq_epi8(_mm_subs_epu8(in, _mm_set1_epi8(0x1F)), zero)); // in < 0x20 + // the quote and the control characters: (c ^ 2) < 33, that is, + // (c ^ 2) - 32 saturates to 0 (see vector_plain_run()) + const __m128i special = _mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('\\')), + _mm_cmpeq_epi8(_mm_subs_epu8(_mm_xor_si128(in, _mm_set1_epi8(2)), _mm_set1_epi8(32)), zero)); const __m128i prev1 = _mm_alignr_epi8(in, prev, 15); const __m128i sc = _mm_and_si128(_mm_and_si128(_mm_shuffle_epi8(t1h, _mm_and_si128(_mm_srli_epi16(prev1, 4), nibble)), _mm_shuffle_epi8(t1l, _mm_and_si128(prev1, nibble))), @@ -37121,8 +37128,10 @@ class serializer const uint8x8_t lo = vld1_u8(reinterpret_cast(p)); const uint8x8_t hi = vld1_u8(reinterpret_cast(p + n - 8)); const uint8x16_t in = vcombine_u8(lo, hi); - const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('"')), vceqq_u8(in, vdupq_n_u8('\\'))), - vorrq_u8(vcltq_s8(vreinterpretq_s8_u8(in), vdupq_n_s8(0x20)), vceqq_u8(in, vdupq_n_u8(0x7F)))); + // (int8_t)(c ^ 2) < 33: the quote, the control characters and the + // non-ASCII bytes (see vector_plain_run()) + const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('\\')), vceqq_u8(in, vdupq_n_u8(0x7F))), + vcltq_s8(vreinterpretq_s8_u8(veorq_u8(in, vdupq_n_u8(2))), vdupq_n_s8(33))); if (vmaxvq_u8(special) != 0) { return false; @@ -37133,8 +37142,8 @@ class serializer const __m128i lo = _mm_loadl_epi64(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) const __m128i hi = _mm_loadl_epi64(static_cast(static_cast(p + n - 8))); // NOLINT(bugprone-casting-through-void) const __m128i in = _mm_unpacklo_epi64(lo, hi); - const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), - _mm_or_si128(_mm_cmplt_epi8(in, _mm_set1_epi8(0x20)), _mm_cmpeq_epi8(in, _mm_set1_epi8(0x7F)))); + const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('\\')), _mm_cmpeq_epi8(in, _mm_set1_epi8(0x7F))), + _mm_cmplt_epi8(_mm_xor_si128(in, _mm_set1_epi8(2)), _mm_set1_epi8(33))); if (_mm_movemask_epi8(special) != 0) { return false; @@ -37153,10 +37162,13 @@ class serializer #if JSON_SCAN_NEON const uint8x16_t a = vld1q_u8(reinterpret_cast(p)); const uint8x16_t b = vld1q_u8(reinterpret_cast(p + n - 16)); - const uint8x16_t special = vorrq_u8(vorrq_u8(vorrq_u8(vceqq_u8(a, vdupq_n_u8('"')), vceqq_u8(a, vdupq_n_u8('\\'))), - vorrq_u8(vceqq_u8(b, vdupq_n_u8('"')), vceqq_u8(b, vdupq_n_u8('\\')))), - vorrq_u8(vorrq_u8(vcltq_s8(vreinterpretq_s8_u8(a), vdupq_n_s8(0x20)), vcltq_s8(vreinterpretq_s8_u8(b), vdupq_n_s8(0x20))), - vorrq_u8(vceqq_u8(a, vdupq_n_u8(0x7F)), vceqq_u8(b, vdupq_n_u8(0x7F))))); + // (int8_t)(c ^ 2) < 33: the quote, the control characters and the + // non-ASCII bytes (see vector_plain_run()) + const uint8x16_t flip = vdupq_n_u8(2); + const int8x16_t limit = vdupq_n_s8(33); + const uint8x16_t special = vorrq_u8(vorrq_u8(vorrq_u8(vceqq_u8(a, vdupq_n_u8('\\')), vceqq_u8(b, vdupq_n_u8('\\'))), + vorrq_u8(vceqq_u8(a, vdupq_n_u8(0x7F)), vceqq_u8(b, vdupq_n_u8(0x7F)))), + vorrq_u8(vcltq_s8(vreinterpretq_s8_u8(veorq_u8(a, flip)), limit), vcltq_s8(vreinterpretq_s8_u8(veorq_u8(b, flip)), limit))); if (vmaxvq_u8(special) != 0) { return false; @@ -37166,14 +37178,15 @@ class serializer #else const __m128i a = _mm_loadu_si128(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) const __m128i b = _mm_loadu_si128(static_cast(static_cast(p + n - 16))); // NOLINT(bugprone-casting-through-void) - const __m128i quote = _mm_set1_epi8('"'); + // (int8_t)(c ^ 2) < 33: the quote, the control characters and the + // non-ASCII bytes (see vector_plain_run()) const __m128i backslash = _mm_set1_epi8('\\'); - const __m128i space = _mm_set1_epi8(0x20); const __m128i del = _mm_set1_epi8(0x7F); - const __m128i special = _mm_or_si128(_mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(a, quote), _mm_cmpeq_epi8(a, backslash)), - _mm_or_si128(_mm_cmpeq_epi8(b, quote), _mm_cmpeq_epi8(b, backslash))), - _mm_or_si128(_mm_or_si128(_mm_cmplt_epi8(a, space), _mm_cmplt_epi8(b, space)), - _mm_or_si128(_mm_cmpeq_epi8(a, del), _mm_cmpeq_epi8(b, del)))); + const __m128i flip = _mm_set1_epi8(2); + const __m128i limit = _mm_set1_epi8(33); + const __m128i special = _mm_or_si128(_mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(a, backslash), _mm_cmpeq_epi8(b, backslash)), + _mm_or_si128(_mm_cmpeq_epi8(a, del), _mm_cmpeq_epi8(b, del))), + _mm_or_si128(_mm_cmplt_epi8(_mm_xor_si128(a, flip), limit), _mm_cmplt_epi8(_mm_xor_si128(b, flip), limit))); if (_mm_movemask_epi8(special) != 0) { return false; diff --git a/single_include/nlohmann/json_view.hpp b/single_include/nlohmann/json_view.hpp index 22c72d982..282898238 100644 --- a/single_include/nlohmann/json_view.hpp +++ b/single_include/nlohmann/json_view.hpp @@ -759,8 +759,10 @@ namespace view character, or not ASCII, 16 bytes per step Stops at such a byte, or where fewer than 16 bytes are left (the caller tells -the two apart). A signed compare with 0x20 finds control characters and -non-ASCII bytes at once. +the two apart). One signed compare finds the quote, the control characters and +the non-ASCII bytes at once: these are exactly the bytes with +(int8_t)(c ^ 2) < 33 (see detail::vector_plain_run()). The backslash needs its +own compare. */ NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* vector_plain_run(const unsigned char* p, const unsigned char* e) noexcept { @@ -768,8 +770,8 @@ NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* vector_plain_run(const unsigned { #if NLOHMANN_VIEW_NEON const uint8x16_t in = vld1q_u8(p); - const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('"')), vceqq_u8(in, vdupq_n_u8('\\'))), - vcltq_s8(vreinterpretq_s8_u8(in), vdupq_n_s8(0x20))); + const uint8x16_t special = vorrq_u8(vceqq_u8(in, vdupq_n_u8('\\')), + vcltq_s8(vreinterpretq_s8_u8(veorq_u8(in, vdupq_n_u8(2))), vdupq_n_s8(33))); // one nibble per byte (the usual NEON replacement of x86's movemask, see // D. Kutenin, "Porting x86 vector bitmask optimizations to Arm NEON", 2022) const std::uint64_t bits = vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(special), 4)), 0); @@ -779,8 +781,8 @@ NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* vector_plain_run(const unsigned } #else const __m128i in = _mm_loadu_si128(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) - const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), - _mm_cmplt_epi8(in, _mm_set1_epi8(0x20))); + const __m128i special = _mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('\\')), + _mm_cmplt_epi8(_mm_xor_si128(in, _mm_set1_epi8(2)), _mm_set1_epi8(33))); const auto bits = static_cast(static_cast(_mm_movemask_epi8(special))); if (bits != 0) { @@ -806,15 +808,17 @@ NLOHMANN_VIEW_ALWAYS_INLINE unsigned copy_plain_run16(const unsigned char* p, ch #if NLOHMANN_VIEW_NEON const uint8x16_t in = vld1q_u8(p); vst1q_u8(reinterpret_cast(w), in); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) - const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('"')), vceqq_u8(in, vdupq_n_u8('\\'))), - vcltq_s8(vreinterpretq_s8_u8(in), vdupq_n_s8(0x20))); + // (int8_t)(c ^ 2) < 33: the quote, the control characters and the + // non-ASCII bytes (see vector_plain_run()) + const uint8x16_t special = vorrq_u8(vceqq_u8(in, vdupq_n_u8('\\')), + vcltq_s8(vreinterpretq_s8_u8(veorq_u8(in, vdupq_n_u8(2))), vdupq_n_s8(33))); const std::uint64_t bits = vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(special), 4)), 0); return bits != 0 ? static_cast(count_trailing_zeros(bits) >> 2u) : 16u; #else const __m128i in = _mm_loadu_si128(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) _mm_storeu_si128(static_cast<__m128i*>(static_cast(w)), in); // NOLINT(bugprone-casting-through-void) - const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), - _mm_cmplt_epi8(in, _mm_set1_epi8(0x20))); + const __m128i special = _mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('\\')), + _mm_cmplt_epi8(_mm_xor_si128(in, _mm_set1_epi8(2)), _mm_set1_epi8(33))); const auto bits = static_cast(static_cast(_mm_movemask_epi8(special))); return bits != 0 ? static_cast(count_trailing_zeros(bits)) : 16u; #endif @@ -974,7 +978,8 @@ NLOHMANN_VIEW_SSSE3_TARGET NLOHMANN_VIEW_NOINLINE NLOHMANN_VIEW_NOCLONE inline c while (e - block >= 16) { const uint8x16_t in = vld1q_u8(block); - const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('"')), vceqq_u8(in, vdupq_n_u8('\\'))), vcltq_u8(in, vdupq_n_u8(0x20))); + // the quote and the control characters: (c ^ 2) < 33 (see vector_plain_run()) + const uint8x16_t special = vorrq_u8(vceqq_u8(in, vdupq_n_u8('\\')), vcltq_u8(veorq_u8(in, vdupq_n_u8(2)), vdupq_n_u8(33))); const uint8x16_t prev1 = vextq_u8(prev, in, 15); const uint8x16_t sc = vandq_u8(vandq_u8(vqtbl1q_u8(t1h, vshrq_n_u8(prev1, 4)), vqtbl1q_u8(t1l, vandq_u8(prev1, vdupq_n_u8(0x0F)))), vqtbl1q_u8(t2h, vshrq_n_u8(in, 4))); const uint8x16_t must23 = vorrq_u8(vqsubq_u8(vextq_u8(prev, in, 14), vdupq_n_u8(0xE0 - 0x80)), vqsubq_u8(vextq_u8(prev, in, 13), vdupq_n_u8(0xF0 - 0x80))); @@ -1013,8 +1018,10 @@ NLOHMANN_VIEW_SSSE3_TARGET NLOHMANN_VIEW_NOINLINE NLOHMANN_VIEW_NOCLONE inline c while (e - block >= 16) { const __m128i in = _mm_loadu_si128(static_cast(static_cast(block))); // NOLINT(bugprone-casting-through-void) - const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), - _mm_cmpeq_epi8(_mm_subs_epu8(in, _mm_set1_epi8(0x1F)), zero)); // in < 0x20 + // the quote and the control characters: (c ^ 2) < 33, that is, + // (c ^ 2) - 32 saturates to 0 (see vector_plain_run()) + const __m128i special = _mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('\\')), + _mm_cmpeq_epi8(_mm_subs_epu8(_mm_xor_si128(in, _mm_set1_epi8(2)), _mm_set1_epi8(32)), zero)); const __m128i prev1 = _mm_alignr_epi8(in, prev, 15); const __m128i sc = _mm_and_si128(_mm_and_si128(_mm_shuffle_epi8(t1h, _mm_and_si128(_mm_srli_epi16(prev1, 4), nibble)), _mm_shuffle_epi8(t1l, _mm_and_si128(prev1, nibble))), @@ -8600,9 +8607,12 @@ inline std::size_t plain_output_run(const unsigned char* s, std::size_t n) noexc for (; i + 16 <= n; i += 16) { const __m128i in = _mm_loadu_si128(static_cast(static_cast(s + i))); // NOLINT(bugprone-casting-through-void) - // (a byte is below 0x20 if it is its own minimum with 0x1F) - const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), - _mm_cmpeq_epi8(_mm_min_epu8(in, _mm_set1_epi8(0x1F)), in)); + // the quote and the control characters are the bytes with + // (c ^ 2) <= 32, that is, those that are their own minimum with 32 + // (see vector_plain_run()) + const __m128i flipped = _mm_xor_si128(in, _mm_set1_epi8(2)); + const __m128i special = _mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('\\')), + _mm_cmpeq_epi8(_mm_min_epu8(flipped, _mm_set1_epi8(32)), flipped)); const auto mask = static_cast(static_cast(_mm_movemask_epi8(special))); if (mask != 0) { @@ -8665,20 +8675,21 @@ inline bool is_ascii_without_delete(const char* s, std::size_t n) noexcept #if NLOHMANN_VIEW_VECTOR /// the bytes of the 16 at p that ensure_ascii cannot copy (a quote, a -/// backslash, a control character, DEL, or not ASCII; a signed compare with -/// 0x20 finds control characters and non-ASCII bytes at once), one bit per -/// byte (x86) or one nibble per byte (NEON) +/// backslash, a control character, DEL, or not ASCII; the signed compare +/// (int8_t)(c ^ 2) < 33 finds the quote, the control characters and the +/// non-ASCII bytes at once, see vector_plain_run()), one bit per byte (x86) or +/// one nibble per byte (NEON) NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t ascii_stop_mask(const unsigned char* p) noexcept { #if NLOHMANN_VIEW_NEON const uint8x16_t in = vld1q_u8(p); - const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('"')), vceqq_u8(in, vdupq_n_u8('\\'))), - vorrq_u8(vceqq_u8(in, vdupq_n_u8(0x7F)), vcltq_s8(vreinterpretq_s8_u8(in), vdupq_n_s8(0x20)))); + const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('\\')), vceqq_u8(in, vdupq_n_u8(0x7F))), + vcltq_s8(vreinterpretq_s8_u8(veorq_u8(in, vdupq_n_u8(2))), vdupq_n_s8(33))); return vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(special), 4)), 0); #else const __m128i in = _mm_loadu_si128(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) - const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), - _mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8(0x7F)), _mm_cmplt_epi8(in, _mm_set1_epi8(0x20)))); + const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('\\')), _mm_cmpeq_epi8(in, _mm_set1_epi8(0x7F))), + _mm_cmplt_epi8(_mm_xor_si128(in, _mm_set1_epi8(2)), _mm_set1_epi8(33))); return static_cast(static_cast(_mm_movemask_epi8(special))); #endif }