diff --git a/include/nlohmann/detail/input/simd_scan.hpp b/include/nlohmann/detail/input/simd_scan.hpp index 6c34421fe..e667c8366 100644 --- a/include/nlohmann/detail/input/simd_scan.hpp +++ b/include/nlohmann/detail/input/simd_scan.hpp @@ -92,8 +92,12 @@ namespace detail character, or not ASCII, 16 bytes per step Stops at such a byte, or where fewer than 16 bytes are left (the caller tells -the two apart). A signed compare with 0x20 finds control characters and -non-ASCII bytes at once. +the two apart). One signed compare finds the quote, the control characters and +the non-ASCII bytes at once: flipping bit 1 maps 0x00-0x1F onto itself and the +quote (0x22) onto 0x20, keeps 0x80-0xFF negative, and maps every other byte to +33 or more, so these are exactly the bytes with (int8_t)(c ^ 2) < 33 (see +D. Lemire, "Detect control characters, quotes and backslashes efficiently +using SWAR", 2025). The backslash needs its own compare. */ JSON_HEDLEY_ALWAYS_INLINE const unsigned char* vector_plain_run(const unsigned char* p, const unsigned char* e) noexcept { @@ -101,8 +105,8 @@ JSON_HEDLEY_ALWAYS_INLINE const unsigned char* vector_plain_run(const unsigned c { #if JSON_SCAN_NEON const uint8x16_t in = vld1q_u8(p); - const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('"')), vceqq_u8(in, vdupq_n_u8('\\'))), - vcltq_s8(vreinterpretq_s8_u8(in), vdupq_n_s8(0x20))); + const uint8x16_t special = vorrq_u8(vceqq_u8(in, vdupq_n_u8('\\')), + vcltq_s8(vreinterpretq_s8_u8(veorq_u8(in, vdupq_n_u8(2))), vdupq_n_s8(33))); // one nibble per byte (the usual NEON replacement of x86's movemask, see // D. Kutenin, "Porting x86 vector bitmask optimizations to Arm NEON", 2022) const std::uint64_t bits = vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(special), 4)), 0); @@ -112,8 +116,8 @@ JSON_HEDLEY_ALWAYS_INLINE const unsigned char* vector_plain_run(const unsigned c } #else const __m128i in = _mm_loadu_si128(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) - const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), - _mm_cmplt_epi8(in, _mm_set1_epi8(0x20))); + const __m128i special = _mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('\\')), + _mm_cmplt_epi8(_mm_xor_si128(in, _mm_set1_epi8(2)), _mm_set1_epi8(33))); const auto bits = static_cast(static_cast(_mm_movemask_epi8(special))); if (bits != 0) { @@ -250,7 +254,8 @@ JSON_SCAN_SSSE3_TARGET JSON_HEDLEY_NEVER_INLINE inline bool vector_utf8_run(cons while (e - block >= 16) { const uint8x16_t in = vld1q_u8(block); - const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('"')), vceqq_u8(in, vdupq_n_u8('\\'))), vcltq_u8(in, vdupq_n_u8(0x20))); + // the quote and the control characters: (c ^ 2) < 33 (see vector_plain_run()) + const uint8x16_t special = vorrq_u8(vceqq_u8(in, vdupq_n_u8('\\')), vcltq_u8(veorq_u8(in, vdupq_n_u8(2)), vdupq_n_u8(33))); const uint8x16_t prev1 = vextq_u8(prev, in, 15); const uint8x16_t sc = vandq_u8(vandq_u8(vqtbl1q_u8(t1h, vshrq_n_u8(prev1, 4)), vqtbl1q_u8(t1l, vandq_u8(prev1, vdupq_n_u8(0x0F)))), vqtbl1q_u8(t2h, vshrq_n_u8(in, 4))); const uint8x16_t must23 = vorrq_u8(vqsubq_u8(vextq_u8(prev, in, 14), vdupq_n_u8(0xE0 - 0x80)), vqsubq_u8(vextq_u8(prev, in, 13), vdupq_n_u8(0xF0 - 0x80))); @@ -290,8 +295,10 @@ JSON_SCAN_SSSE3_TARGET JSON_HEDLEY_NEVER_INLINE inline bool vector_utf8_run(cons while (e - block >= 16) { const __m128i in = _mm_loadu_si128(static_cast(static_cast(block))); // NOLINT(bugprone-casting-through-void) - const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), - _mm_cmpeq_epi8(_mm_subs_epu8(in, _mm_set1_epi8(0x1F)), zero)); // in < 0x20 + // the quote and the control characters: (c ^ 2) < 33, that is, + // (c ^ 2) - 32 saturates to 0 (see vector_plain_run()) + const __m128i special = _mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('\\')), + _mm_cmpeq_epi8(_mm_subs_epu8(_mm_xor_si128(in, _mm_set1_epi8(2)), _mm_set1_epi8(32)), zero)); const __m128i prev1 = _mm_alignr_epi8(in, prev, 15); const __m128i sc = _mm_and_si128(_mm_and_si128(_mm_shuffle_epi8(t1h, _mm_and_si128(_mm_srli_epi16(prev1, 4), nibble)), _mm_shuffle_epi8(t1l, _mm_and_si128(prev1, nibble))), diff --git a/include/nlohmann/detail/input/string_scan.hpp b/include/nlohmann/detail/input/string_scan.hpp index 8564746ce..fd4e03097 100644 --- a/include/nlohmann/detail/input/string_scan.hpp +++ b/include/nlohmann/detail/input/string_scan.hpp @@ -58,11 +58,10 @@ inline std::uint64_t swar_string_special(std::uint64_t v) noexcept constexpr std::uint64_t high = 0x8080808080808080ull; const std::uint64_t q = v ^ 0x2222222222222222ull; // '"' (0x22) const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; // '\\' (0x5C) - const std::uint64_t has_quote = (q - ones) & ~q & high; - const std::uint64_t has_backslash = (b - ones) & ~b & high; - const std::uint64_t has_control = (v - 0x2020202020202020ull) & ~v & high; // < 0x20 - const std::uint64_t has_non_ascii = v & high; // >= 0x80 - return has_quote | has_backslash | has_control | has_non_ascii; + // The usual test for a zero byte is (x - ones) & ~x & high. The `& ~x` + // only removes bytes from 0x80, and `v & high` flags those anyway, so it + // can be dropped. Clang finds this reduction itself; GCC does not. + return ((q - ones) | (b - ones) | (v - 0x2020202020202020ull) | v) & high; } // return the index of the first is_string_special() byte in [data, data+n), or @@ -143,11 +142,8 @@ inline std::size_t find_ascii_copyable_run(const unsigned char* data, std::size_ const std::uint64_t q = v ^ 0x2222222222222222ull; // '"' (0x22) const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; // '\\' (0x5C) const std::uint64_t d = v ^ 0x7F7F7F7F7F7F7F7Full; // DEL (0x7F) - const std::uint64_t stop = ((q - ones) & ~q & high) // == '"' - | ((b - ones) & ~b & high) // == '\\' - | ((d - ones) & ~d & high) // == 0x7F - | ((v - 0x2020202020202020ull) & ~v & high) // < 0x20 - | (v & high); // >= 0x80 + // (see swar_string_special(): the bytes from 0x80 are stops anyway) + const std::uint64_t stop = ((q - ones) | (b - ones) | (d - ones) | (v - 0x2020202020202020ull) | v) & high; if (stop != 0) { // the lowest flagged byte is the first one to stop at (see diff --git a/include/nlohmann/detail/output/serializer.hpp b/include/nlohmann/detail/output/serializer.hpp index 235e7bafd..a2ba87205 100644 --- a/include/nlohmann/detail/output/serializer.hpp +++ b/include/nlohmann/detail/output/serializer.hpp @@ -724,10 +724,8 @@ class serializer constexpr std::uint64_t ones = 0x0101010101010101ull; const std::uint64_t q = v ^ 0x2222222222222222ull; // '"' const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; // '\\' - return (((q - ones) & ~q) | ((b - ones) & ~b) // == '"', == '\\' - | ((v - 0x2020202020202020ull) & ~v) // < 0x20 - | v | (v + ones)) // >= 0x7F - & 0x8080808080808080ull; + // (see swar_string_special(): the bytes from 0x80 are flagged anyway) + return ((q - ones) | (b - ones) | (v - 0x2020202020202020ull) | v | (v + ones)) & 0x8080808080808080ull; } template @@ -836,8 +834,10 @@ class serializer const uint8x8_t lo = vld1_u8(reinterpret_cast(p)); const uint8x8_t hi = vld1_u8(reinterpret_cast(p + n - 8)); const uint8x16_t in = vcombine_u8(lo, hi); - const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('"')), vceqq_u8(in, vdupq_n_u8('\\'))), - vorrq_u8(vcltq_s8(vreinterpretq_s8_u8(in), vdupq_n_s8(0x20)), vceqq_u8(in, vdupq_n_u8(0x7F)))); + // (int8_t)(c ^ 2) < 33: the quote, the control characters and the + // non-ASCII bytes (see vector_plain_run()) + const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('\\')), vceqq_u8(in, vdupq_n_u8(0x7F))), + vcltq_s8(vreinterpretq_s8_u8(veorq_u8(in, vdupq_n_u8(2))), vdupq_n_s8(33))); if (vmaxvq_u8(special) != 0) { return false; @@ -848,8 +848,8 @@ class serializer const __m128i lo = _mm_loadl_epi64(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) const __m128i hi = _mm_loadl_epi64(static_cast(static_cast(p + n - 8))); // NOLINT(bugprone-casting-through-void) const __m128i in = _mm_unpacklo_epi64(lo, hi); - const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), - _mm_or_si128(_mm_cmplt_epi8(in, _mm_set1_epi8(0x20)), _mm_cmpeq_epi8(in, _mm_set1_epi8(0x7F)))); + const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('\\')), _mm_cmpeq_epi8(in, _mm_set1_epi8(0x7F))), + _mm_cmplt_epi8(_mm_xor_si128(in, _mm_set1_epi8(2)), _mm_set1_epi8(33))); if (_mm_movemask_epi8(special) != 0) { return false; @@ -868,10 +868,13 @@ class serializer #if JSON_SCAN_NEON const uint8x16_t a = vld1q_u8(reinterpret_cast(p)); const uint8x16_t b = vld1q_u8(reinterpret_cast(p + n - 16)); - const uint8x16_t special = vorrq_u8(vorrq_u8(vorrq_u8(vceqq_u8(a, vdupq_n_u8('"')), vceqq_u8(a, vdupq_n_u8('\\'))), - vorrq_u8(vceqq_u8(b, vdupq_n_u8('"')), vceqq_u8(b, vdupq_n_u8('\\')))), - vorrq_u8(vorrq_u8(vcltq_s8(vreinterpretq_s8_u8(a), vdupq_n_s8(0x20)), vcltq_s8(vreinterpretq_s8_u8(b), vdupq_n_s8(0x20))), - vorrq_u8(vceqq_u8(a, vdupq_n_u8(0x7F)), vceqq_u8(b, vdupq_n_u8(0x7F))))); + // (int8_t)(c ^ 2) < 33: the quote, the control characters and the + // non-ASCII bytes (see vector_plain_run()) + const uint8x16_t flip = vdupq_n_u8(2); + const int8x16_t limit = vdupq_n_s8(33); + const uint8x16_t special = vorrq_u8(vorrq_u8(vorrq_u8(vceqq_u8(a, vdupq_n_u8('\\')), vceqq_u8(b, vdupq_n_u8('\\'))), + vorrq_u8(vceqq_u8(a, vdupq_n_u8(0x7F)), vceqq_u8(b, vdupq_n_u8(0x7F)))), + vorrq_u8(vcltq_s8(vreinterpretq_s8_u8(veorq_u8(a, flip)), limit), vcltq_s8(vreinterpretq_s8_u8(veorq_u8(b, flip)), limit))); if (vmaxvq_u8(special) != 0) { return false; @@ -881,14 +884,15 @@ class serializer #else const __m128i a = _mm_loadu_si128(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) const __m128i b = _mm_loadu_si128(static_cast(static_cast(p + n - 16))); // NOLINT(bugprone-casting-through-void) - const __m128i quote = _mm_set1_epi8('"'); + // (int8_t)(c ^ 2) < 33: the quote, the control characters and the + // non-ASCII bytes (see vector_plain_run()) const __m128i backslash = _mm_set1_epi8('\\'); - const __m128i space = _mm_set1_epi8(0x20); const __m128i del = _mm_set1_epi8(0x7F); - const __m128i special = _mm_or_si128(_mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(a, quote), _mm_cmpeq_epi8(a, backslash)), - _mm_or_si128(_mm_cmpeq_epi8(b, quote), _mm_cmpeq_epi8(b, backslash))), - _mm_or_si128(_mm_or_si128(_mm_cmplt_epi8(a, space), _mm_cmplt_epi8(b, space)), - _mm_or_si128(_mm_cmpeq_epi8(a, del), _mm_cmpeq_epi8(b, del)))); + const __m128i flip = _mm_set1_epi8(2); + const __m128i limit = _mm_set1_epi8(33); + const __m128i special = _mm_or_si128(_mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(a, backslash), _mm_cmpeq_epi8(b, backslash)), + _mm_or_si128(_mm_cmpeq_epi8(a, del), _mm_cmpeq_epi8(b, del))), + _mm_or_si128(_mm_cmplt_epi8(_mm_xor_si128(a, flip), limit), _mm_cmplt_epi8(_mm_xor_si128(b, flip), limit))); if (_mm_movemask_epi8(special) != 0) { return false; diff --git a/include/nlohmann/detail/view/serializer.hpp b/include/nlohmann/detail/view/serializer.hpp index 1c965f78f..068c8af52 100644 --- a/include/nlohmann/detail/view/serializer.hpp +++ b/include/nlohmann/detail/view/serializer.hpp @@ -152,9 +152,12 @@ inline std::size_t plain_output_run(const unsigned char* s, std::size_t n) noexc for (; i + 16 <= n; i += 16) { const __m128i in = _mm_loadu_si128(static_cast(static_cast(s + i))); // NOLINT(bugprone-casting-through-void) - // (a byte is below 0x20 if it is its own minimum with 0x1F) - const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), - _mm_cmpeq_epi8(_mm_min_epu8(in, _mm_set1_epi8(0x1F)), in)); + // the quote and the control characters are the bytes with + // (c ^ 2) <= 32, that is, those that are their own minimum with 32 + // (see vector_plain_run()) + const __m128i flipped = _mm_xor_si128(in, _mm_set1_epi8(2)); + const __m128i special = _mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('\\')), + _mm_cmpeq_epi8(_mm_min_epu8(flipped, _mm_set1_epi8(32)), flipped)); const auto mask = static_cast(static_cast(_mm_movemask_epi8(special))); if (mask != 0) { @@ -217,20 +220,21 @@ inline bool is_ascii_without_delete(const char* s, std::size_t n) noexcept #if NLOHMANN_VIEW_VECTOR /// the bytes of the 16 at p that ensure_ascii cannot copy (a quote, a -/// backslash, a control character, DEL, or not ASCII; a signed compare with -/// 0x20 finds control characters and non-ASCII bytes at once), one bit per -/// byte (x86) or one nibble per byte (NEON) +/// backslash, a control character, DEL, or not ASCII; the signed compare +/// (int8_t)(c ^ 2) < 33 finds the quote, the control characters and the +/// non-ASCII bytes at once, see vector_plain_run()), one bit per byte (x86) or +/// one nibble per byte (NEON) NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t ascii_stop_mask(const unsigned char* p) noexcept { #if NLOHMANN_VIEW_NEON const uint8x16_t in = vld1q_u8(p); - const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('"')), vceqq_u8(in, vdupq_n_u8('\\'))), - vorrq_u8(vceqq_u8(in, vdupq_n_u8(0x7F)), vcltq_s8(vreinterpretq_s8_u8(in), vdupq_n_s8(0x20)))); + const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('\\')), vceqq_u8(in, vdupq_n_u8(0x7F))), + vcltq_s8(vreinterpretq_s8_u8(veorq_u8(in, vdupq_n_u8(2))), vdupq_n_s8(33))); return vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(special), 4)), 0); #else const __m128i in = _mm_loadu_si128(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) - const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), - _mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8(0x7F)), _mm_cmplt_epi8(in, _mm_set1_epi8(0x20)))); + const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('\\')), _mm_cmpeq_epi8(in, _mm_set1_epi8(0x7F))), + _mm_cmplt_epi8(_mm_xor_si128(in, _mm_set1_epi8(2)), _mm_set1_epi8(33))); return static_cast(static_cast(_mm_movemask_epi8(special))); #endif } diff --git a/include/nlohmann/detail/view/simd.hpp b/include/nlohmann/detail/view/simd.hpp index f4745f6b8..b1fc8ba74 100644 --- a/include/nlohmann/detail/view/simd.hpp +++ b/include/nlohmann/detail/view/simd.hpp @@ -91,8 +91,10 @@ namespace view character, or not ASCII, 16 bytes per step Stops at such a byte, or where fewer than 16 bytes are left (the caller tells -the two apart). A signed compare with 0x20 finds control characters and -non-ASCII bytes at once. +the two apart). One signed compare finds the quote, the control characters and +the non-ASCII bytes at once: these are exactly the bytes with +(int8_t)(c ^ 2) < 33 (see detail::vector_plain_run()). The backslash needs its +own compare. */ NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* vector_plain_run(const unsigned char* p, const unsigned char* e) noexcept { @@ -100,8 +102,8 @@ NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* vector_plain_run(const unsigned { #if NLOHMANN_VIEW_NEON const uint8x16_t in = vld1q_u8(p); - const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('"')), vceqq_u8(in, vdupq_n_u8('\\'))), - vcltq_s8(vreinterpretq_s8_u8(in), vdupq_n_s8(0x20))); + const uint8x16_t special = vorrq_u8(vceqq_u8(in, vdupq_n_u8('\\')), + vcltq_s8(vreinterpretq_s8_u8(veorq_u8(in, vdupq_n_u8(2))), vdupq_n_s8(33))); // one nibble per byte (the usual NEON replacement of x86's movemask, see // D. Kutenin, "Porting x86 vector bitmask optimizations to Arm NEON", 2022) const std::uint64_t bits = vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(special), 4)), 0); @@ -111,8 +113,8 @@ NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* vector_plain_run(const unsigned } #else const __m128i in = _mm_loadu_si128(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) - const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), - _mm_cmplt_epi8(in, _mm_set1_epi8(0x20))); + const __m128i special = _mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('\\')), + _mm_cmplt_epi8(_mm_xor_si128(in, _mm_set1_epi8(2)), _mm_set1_epi8(33))); const auto bits = static_cast(static_cast(_mm_movemask_epi8(special))); if (bits != 0) { @@ -138,15 +140,17 @@ NLOHMANN_VIEW_ALWAYS_INLINE unsigned copy_plain_run16(const unsigned char* p, ch #if NLOHMANN_VIEW_NEON const uint8x16_t in = vld1q_u8(p); vst1q_u8(reinterpret_cast(w), in); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) - const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('"')), vceqq_u8(in, vdupq_n_u8('\\'))), - vcltq_s8(vreinterpretq_s8_u8(in), vdupq_n_s8(0x20))); + // (int8_t)(c ^ 2) < 33: the quote, the control characters and the + // non-ASCII bytes (see vector_plain_run()) + const uint8x16_t special = vorrq_u8(vceqq_u8(in, vdupq_n_u8('\\')), + vcltq_s8(vreinterpretq_s8_u8(veorq_u8(in, vdupq_n_u8(2))), vdupq_n_s8(33))); const std::uint64_t bits = vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(special), 4)), 0); return bits != 0 ? static_cast(count_trailing_zeros(bits) >> 2u) : 16u; #else const __m128i in = _mm_loadu_si128(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) _mm_storeu_si128(static_cast<__m128i*>(static_cast(w)), in); // NOLINT(bugprone-casting-through-void) - const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), - _mm_cmplt_epi8(in, _mm_set1_epi8(0x20))); + const __m128i special = _mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('\\')), + _mm_cmplt_epi8(_mm_xor_si128(in, _mm_set1_epi8(2)), _mm_set1_epi8(33))); const auto bits = static_cast(static_cast(_mm_movemask_epi8(special))); return bits != 0 ? static_cast(count_trailing_zeros(bits)) : 16u; #endif @@ -306,7 +310,8 @@ NLOHMANN_VIEW_SSSE3_TARGET NLOHMANN_VIEW_NOINLINE NLOHMANN_VIEW_NOCLONE inline c while (e - block >= 16) { const uint8x16_t in = vld1q_u8(block); - const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('"')), vceqq_u8(in, vdupq_n_u8('\\'))), vcltq_u8(in, vdupq_n_u8(0x20))); + // the quote and the control characters: (c ^ 2) < 33 (see vector_plain_run()) + const uint8x16_t special = vorrq_u8(vceqq_u8(in, vdupq_n_u8('\\')), vcltq_u8(veorq_u8(in, vdupq_n_u8(2)), vdupq_n_u8(33))); const uint8x16_t prev1 = vextq_u8(prev, in, 15); const uint8x16_t sc = vandq_u8(vandq_u8(vqtbl1q_u8(t1h, vshrq_n_u8(prev1, 4)), vqtbl1q_u8(t1l, vandq_u8(prev1, vdupq_n_u8(0x0F)))), vqtbl1q_u8(t2h, vshrq_n_u8(in, 4))); const uint8x16_t must23 = vorrq_u8(vqsubq_u8(vextq_u8(prev, in, 14), vdupq_n_u8(0xE0 - 0x80)), vqsubq_u8(vextq_u8(prev, in, 13), vdupq_n_u8(0xF0 - 0x80))); @@ -345,8 +350,10 @@ NLOHMANN_VIEW_SSSE3_TARGET NLOHMANN_VIEW_NOINLINE NLOHMANN_VIEW_NOCLONE inline c while (e - block >= 16) { const __m128i in = _mm_loadu_si128(static_cast(static_cast(block))); // NOLINT(bugprone-casting-through-void) - const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), - _mm_cmpeq_epi8(_mm_subs_epu8(in, _mm_set1_epi8(0x1F)), zero)); // in < 0x20 + // the quote and the control characters: (c ^ 2) < 33, that is, + // (c ^ 2) - 32 saturates to 0 (see vector_plain_run()) + const __m128i special = _mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('\\')), + _mm_cmpeq_epi8(_mm_subs_epu8(_mm_xor_si128(in, _mm_set1_epi8(2)), _mm_set1_epi8(32)), zero)); const __m128i prev1 = _mm_alignr_epi8(in, prev, 15); const __m128i sc = _mm_and_si128(_mm_and_si128(_mm_shuffle_epi8(t1h, _mm_and_si128(_mm_srli_epi16(prev1, 4), nibble)), _mm_shuffle_epi8(t1l, _mm_and_si128(prev1, nibble))), diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 3ef7ac4dd..5104551ea 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -12361,8 +12361,12 @@ namespace detail character, or not ASCII, 16 bytes per step Stops at such a byte, or where fewer than 16 bytes are left (the caller tells -the two apart). A signed compare with 0x20 finds control characters and -non-ASCII bytes at once. +the two apart). One signed compare finds the quote, the control characters and +the non-ASCII bytes at once: flipping bit 1 maps 0x00-0x1F onto itself and the +quote (0x22) onto 0x20, keeps 0x80-0xFF negative, and maps every other byte to +33 or more, so these are exactly the bytes with (int8_t)(c ^ 2) < 33 (see +D. Lemire, "Detect control characters, quotes and backslashes efficiently +using SWAR", 2025). The backslash needs its own compare. */ JSON_HEDLEY_ALWAYS_INLINE const unsigned char* vector_plain_run(const unsigned char* p, const unsigned char* e) noexcept { @@ -12370,8 +12374,8 @@ JSON_HEDLEY_ALWAYS_INLINE const unsigned char* vector_plain_run(const unsigned c { #if JSON_SCAN_NEON const uint8x16_t in = vld1q_u8(p); - const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('"')), vceqq_u8(in, vdupq_n_u8('\\'))), - vcltq_s8(vreinterpretq_s8_u8(in), vdupq_n_s8(0x20))); + const uint8x16_t special = vorrq_u8(vceqq_u8(in, vdupq_n_u8('\\')), + vcltq_s8(vreinterpretq_s8_u8(veorq_u8(in, vdupq_n_u8(2))), vdupq_n_s8(33))); // one nibble per byte (the usual NEON replacement of x86's movemask, see // D. Kutenin, "Porting x86 vector bitmask optimizations to Arm NEON", 2022) const std::uint64_t bits = vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(special), 4)), 0); @@ -12381,8 +12385,8 @@ JSON_HEDLEY_ALWAYS_INLINE const unsigned char* vector_plain_run(const unsigned c } #else const __m128i in = _mm_loadu_si128(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) - const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), - _mm_cmplt_epi8(in, _mm_set1_epi8(0x20))); + const __m128i special = _mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('\\')), + _mm_cmplt_epi8(_mm_xor_si128(in, _mm_set1_epi8(2)), _mm_set1_epi8(33))); const auto bits = static_cast(static_cast(_mm_movemask_epi8(special))); if (bits != 0) { @@ -12519,7 +12523,8 @@ JSON_SCAN_SSSE3_TARGET JSON_HEDLEY_NEVER_INLINE inline bool vector_utf8_run(cons while (e - block >= 16) { const uint8x16_t in = vld1q_u8(block); - const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('"')), vceqq_u8(in, vdupq_n_u8('\\'))), vcltq_u8(in, vdupq_n_u8(0x20))); + // the quote and the control characters: (c ^ 2) < 33 (see vector_plain_run()) + const uint8x16_t special = vorrq_u8(vceqq_u8(in, vdupq_n_u8('\\')), vcltq_u8(veorq_u8(in, vdupq_n_u8(2)), vdupq_n_u8(33))); const uint8x16_t prev1 = vextq_u8(prev, in, 15); const uint8x16_t sc = vandq_u8(vandq_u8(vqtbl1q_u8(t1h, vshrq_n_u8(prev1, 4)), vqtbl1q_u8(t1l, vandq_u8(prev1, vdupq_n_u8(0x0F)))), vqtbl1q_u8(t2h, vshrq_n_u8(in, 4))); const uint8x16_t must23 = vorrq_u8(vqsubq_u8(vextq_u8(prev, in, 14), vdupq_n_u8(0xE0 - 0x80)), vqsubq_u8(vextq_u8(prev, in, 13), vdupq_n_u8(0xF0 - 0x80))); @@ -12559,8 +12564,10 @@ JSON_SCAN_SSSE3_TARGET JSON_HEDLEY_NEVER_INLINE inline bool vector_utf8_run(cons while (e - block >= 16) { const __m128i in = _mm_loadu_si128(static_cast(static_cast(block))); // NOLINT(bugprone-casting-through-void) - const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), - _mm_cmpeq_epi8(_mm_subs_epu8(in, _mm_set1_epi8(0x1F)), zero)); // in < 0x20 + // the quote and the control characters: (c ^ 2) < 33, that is, + // (c ^ 2) - 32 saturates to 0 (see vector_plain_run()) + const __m128i special = _mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('\\')), + _mm_cmpeq_epi8(_mm_subs_epu8(_mm_xor_si128(in, _mm_set1_epi8(2)), _mm_set1_epi8(32)), zero)); const __m128i prev1 = _mm_alignr_epi8(in, prev, 15); const __m128i sc = _mm_and_si128(_mm_and_si128(_mm_shuffle_epi8(t1h, _mm_and_si128(_mm_srli_epi16(prev1, 4), nibble)), _mm_shuffle_epi8(t1l, _mm_and_si128(prev1, nibble))), @@ -12712,11 +12719,10 @@ inline std::uint64_t swar_string_special(std::uint64_t v) noexcept constexpr std::uint64_t high = 0x8080808080808080ull; const std::uint64_t q = v ^ 0x2222222222222222ull; // '"' (0x22) const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; // '\\' (0x5C) - const std::uint64_t has_quote = (q - ones) & ~q & high; - const std::uint64_t has_backslash = (b - ones) & ~b & high; - const std::uint64_t has_control = (v - 0x2020202020202020ull) & ~v & high; // < 0x20 - const std::uint64_t has_non_ascii = v & high; // >= 0x80 - return has_quote | has_backslash | has_control | has_non_ascii; + // The usual test for a zero byte is (x - ones) & ~x & high. The `& ~x` + // only removes bytes from 0x80, and `v & high` flags those anyway, so it + // can be dropped. Clang finds this reduction itself; GCC does not. + return ((q - ones) | (b - ones) | (v - 0x2020202020202020ull) | v) & high; } // return the index of the first is_string_special() byte in [data, data+n), or @@ -12797,11 +12803,8 @@ inline std::size_t find_ascii_copyable_run(const unsigned char* data, std::size_ const std::uint64_t q = v ^ 0x2222222222222222ull; // '"' (0x22) const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; // '\\' (0x5C) const std::uint64_t d = v ^ 0x7F7F7F7F7F7F7F7Full; // DEL (0x7F) - const std::uint64_t stop = ((q - ones) & ~q & high) // == '"' - | ((b - ones) & ~b & high) // == '\\' - | ((d - ones) & ~d & high) // == 0x7F - | ((v - 0x2020202020202020ull) & ~v & high) // < 0x20 - | (v & high); // >= 0x80 + // (see swar_string_special(): the bytes from 0x80 are stops anyway) + const std::uint64_t stop = ((q - ones) | (b - ones) | (d - ones) | (v - 0x2020202020202020ull) | v) & high; if (stop != 0) { // the lowest flagged byte is the first one to stop at (see @@ -37238,10 +37241,8 @@ class serializer constexpr std::uint64_t ones = 0x0101010101010101ull; const std::uint64_t q = v ^ 0x2222222222222222ull; // '"' const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; // '\\' - return (((q - ones) & ~q) | ((b - ones) & ~b) // == '"', == '\\' - | ((v - 0x2020202020202020ull) & ~v) // < 0x20 - | v | (v + ones)) // >= 0x7F - & 0x8080808080808080ull; + // (see swar_string_special(): the bytes from 0x80 are flagged anyway) + return ((q - ones) | (b - ones) | (v - 0x2020202020202020ull) | v | (v + ones)) & 0x8080808080808080ull; } template @@ -37350,8 +37351,10 @@ class serializer const uint8x8_t lo = vld1_u8(reinterpret_cast(p)); const uint8x8_t hi = vld1_u8(reinterpret_cast(p + n - 8)); const uint8x16_t in = vcombine_u8(lo, hi); - const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('"')), vceqq_u8(in, vdupq_n_u8('\\'))), - vorrq_u8(vcltq_s8(vreinterpretq_s8_u8(in), vdupq_n_s8(0x20)), vceqq_u8(in, vdupq_n_u8(0x7F)))); + // (int8_t)(c ^ 2) < 33: the quote, the control characters and the + // non-ASCII bytes (see vector_plain_run()) + const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('\\')), vceqq_u8(in, vdupq_n_u8(0x7F))), + vcltq_s8(vreinterpretq_s8_u8(veorq_u8(in, vdupq_n_u8(2))), vdupq_n_s8(33))); if (vmaxvq_u8(special) != 0) { return false; @@ -37362,8 +37365,8 @@ class serializer const __m128i lo = _mm_loadl_epi64(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) const __m128i hi = _mm_loadl_epi64(static_cast(static_cast(p + n - 8))); // NOLINT(bugprone-casting-through-void) const __m128i in = _mm_unpacklo_epi64(lo, hi); - const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), - _mm_or_si128(_mm_cmplt_epi8(in, _mm_set1_epi8(0x20)), _mm_cmpeq_epi8(in, _mm_set1_epi8(0x7F)))); + const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('\\')), _mm_cmpeq_epi8(in, _mm_set1_epi8(0x7F))), + _mm_cmplt_epi8(_mm_xor_si128(in, _mm_set1_epi8(2)), _mm_set1_epi8(33))); if (_mm_movemask_epi8(special) != 0) { return false; @@ -37382,10 +37385,13 @@ class serializer #if JSON_SCAN_NEON const uint8x16_t a = vld1q_u8(reinterpret_cast(p)); const uint8x16_t b = vld1q_u8(reinterpret_cast(p + n - 16)); - const uint8x16_t special = vorrq_u8(vorrq_u8(vorrq_u8(vceqq_u8(a, vdupq_n_u8('"')), vceqq_u8(a, vdupq_n_u8('\\'))), - vorrq_u8(vceqq_u8(b, vdupq_n_u8('"')), vceqq_u8(b, vdupq_n_u8('\\')))), - vorrq_u8(vorrq_u8(vcltq_s8(vreinterpretq_s8_u8(a), vdupq_n_s8(0x20)), vcltq_s8(vreinterpretq_s8_u8(b), vdupq_n_s8(0x20))), - vorrq_u8(vceqq_u8(a, vdupq_n_u8(0x7F)), vceqq_u8(b, vdupq_n_u8(0x7F))))); + // (int8_t)(c ^ 2) < 33: the quote, the control characters and the + // non-ASCII bytes (see vector_plain_run()) + const uint8x16_t flip = vdupq_n_u8(2); + const int8x16_t limit = vdupq_n_s8(33); + const uint8x16_t special = vorrq_u8(vorrq_u8(vorrq_u8(vceqq_u8(a, vdupq_n_u8('\\')), vceqq_u8(b, vdupq_n_u8('\\'))), + vorrq_u8(vceqq_u8(a, vdupq_n_u8(0x7F)), vceqq_u8(b, vdupq_n_u8(0x7F)))), + vorrq_u8(vcltq_s8(vreinterpretq_s8_u8(veorq_u8(a, flip)), limit), vcltq_s8(vreinterpretq_s8_u8(veorq_u8(b, flip)), limit))); if (vmaxvq_u8(special) != 0) { return false; @@ -37395,14 +37401,15 @@ class serializer #else const __m128i a = _mm_loadu_si128(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) const __m128i b = _mm_loadu_si128(static_cast(static_cast(p + n - 16))); // NOLINT(bugprone-casting-through-void) - const __m128i quote = _mm_set1_epi8('"'); + // (int8_t)(c ^ 2) < 33: the quote, the control characters and the + // non-ASCII bytes (see vector_plain_run()) const __m128i backslash = _mm_set1_epi8('\\'); - const __m128i space = _mm_set1_epi8(0x20); const __m128i del = _mm_set1_epi8(0x7F); - const __m128i special = _mm_or_si128(_mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(a, quote), _mm_cmpeq_epi8(a, backslash)), - _mm_or_si128(_mm_cmpeq_epi8(b, quote), _mm_cmpeq_epi8(b, backslash))), - _mm_or_si128(_mm_or_si128(_mm_cmplt_epi8(a, space), _mm_cmplt_epi8(b, space)), - _mm_or_si128(_mm_cmpeq_epi8(a, del), _mm_cmpeq_epi8(b, del)))); + const __m128i flip = _mm_set1_epi8(2); + const __m128i limit = _mm_set1_epi8(33); + const __m128i special = _mm_or_si128(_mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(a, backslash), _mm_cmpeq_epi8(b, backslash)), + _mm_or_si128(_mm_cmpeq_epi8(a, del), _mm_cmpeq_epi8(b, del))), + _mm_or_si128(_mm_cmplt_epi8(_mm_xor_si128(a, flip), limit), _mm_cmplt_epi8(_mm_xor_si128(b, flip), limit))); if (_mm_movemask_epi8(special) != 0) { return false; diff --git a/single_include/nlohmann/json_view.hpp b/single_include/nlohmann/json_view.hpp index 075544088..f69d1ae43 100644 --- a/single_include/nlohmann/json_view.hpp +++ b/single_include/nlohmann/json_view.hpp @@ -759,8 +759,10 @@ namespace view character, or not ASCII, 16 bytes per step Stops at such a byte, or where fewer than 16 bytes are left (the caller tells -the two apart). A signed compare with 0x20 finds control characters and -non-ASCII bytes at once. +the two apart). One signed compare finds the quote, the control characters and +the non-ASCII bytes at once: these are exactly the bytes with +(int8_t)(c ^ 2) < 33 (see detail::vector_plain_run()). The backslash needs its +own compare. */ NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* vector_plain_run(const unsigned char* p, const unsigned char* e) noexcept { @@ -768,8 +770,8 @@ NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* vector_plain_run(const unsigned { #if NLOHMANN_VIEW_NEON const uint8x16_t in = vld1q_u8(p); - const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('"')), vceqq_u8(in, vdupq_n_u8('\\'))), - vcltq_s8(vreinterpretq_s8_u8(in), vdupq_n_s8(0x20))); + const uint8x16_t special = vorrq_u8(vceqq_u8(in, vdupq_n_u8('\\')), + vcltq_s8(vreinterpretq_s8_u8(veorq_u8(in, vdupq_n_u8(2))), vdupq_n_s8(33))); // one nibble per byte (the usual NEON replacement of x86's movemask, see // D. Kutenin, "Porting x86 vector bitmask optimizations to Arm NEON", 2022) const std::uint64_t bits = vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(special), 4)), 0); @@ -779,8 +781,8 @@ NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* vector_plain_run(const unsigned } #else const __m128i in = _mm_loadu_si128(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) - const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), - _mm_cmplt_epi8(in, _mm_set1_epi8(0x20))); + const __m128i special = _mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('\\')), + _mm_cmplt_epi8(_mm_xor_si128(in, _mm_set1_epi8(2)), _mm_set1_epi8(33))); const auto bits = static_cast(static_cast(_mm_movemask_epi8(special))); if (bits != 0) { @@ -806,15 +808,17 @@ NLOHMANN_VIEW_ALWAYS_INLINE unsigned copy_plain_run16(const unsigned char* p, ch #if NLOHMANN_VIEW_NEON const uint8x16_t in = vld1q_u8(p); vst1q_u8(reinterpret_cast(w), in); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) - const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('"')), vceqq_u8(in, vdupq_n_u8('\\'))), - vcltq_s8(vreinterpretq_s8_u8(in), vdupq_n_s8(0x20))); + // (int8_t)(c ^ 2) < 33: the quote, the control characters and the + // non-ASCII bytes (see vector_plain_run()) + const uint8x16_t special = vorrq_u8(vceqq_u8(in, vdupq_n_u8('\\')), + vcltq_s8(vreinterpretq_s8_u8(veorq_u8(in, vdupq_n_u8(2))), vdupq_n_s8(33))); const std::uint64_t bits = vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(special), 4)), 0); return bits != 0 ? static_cast(count_trailing_zeros(bits) >> 2u) : 16u; #else const __m128i in = _mm_loadu_si128(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) _mm_storeu_si128(static_cast<__m128i*>(static_cast(w)), in); // NOLINT(bugprone-casting-through-void) - const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), - _mm_cmplt_epi8(in, _mm_set1_epi8(0x20))); + const __m128i special = _mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('\\')), + _mm_cmplt_epi8(_mm_xor_si128(in, _mm_set1_epi8(2)), _mm_set1_epi8(33))); const auto bits = static_cast(static_cast(_mm_movemask_epi8(special))); return bits != 0 ? static_cast(count_trailing_zeros(bits)) : 16u; #endif @@ -974,7 +978,8 @@ NLOHMANN_VIEW_SSSE3_TARGET NLOHMANN_VIEW_NOINLINE NLOHMANN_VIEW_NOCLONE inline c while (e - block >= 16) { const uint8x16_t in = vld1q_u8(block); - const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('"')), vceqq_u8(in, vdupq_n_u8('\\'))), vcltq_u8(in, vdupq_n_u8(0x20))); + // the quote and the control characters: (c ^ 2) < 33 (see vector_plain_run()) + const uint8x16_t special = vorrq_u8(vceqq_u8(in, vdupq_n_u8('\\')), vcltq_u8(veorq_u8(in, vdupq_n_u8(2)), vdupq_n_u8(33))); const uint8x16_t prev1 = vextq_u8(prev, in, 15); const uint8x16_t sc = vandq_u8(vandq_u8(vqtbl1q_u8(t1h, vshrq_n_u8(prev1, 4)), vqtbl1q_u8(t1l, vandq_u8(prev1, vdupq_n_u8(0x0F)))), vqtbl1q_u8(t2h, vshrq_n_u8(in, 4))); const uint8x16_t must23 = vorrq_u8(vqsubq_u8(vextq_u8(prev, in, 14), vdupq_n_u8(0xE0 - 0x80)), vqsubq_u8(vextq_u8(prev, in, 13), vdupq_n_u8(0xF0 - 0x80))); @@ -1013,8 +1018,10 @@ NLOHMANN_VIEW_SSSE3_TARGET NLOHMANN_VIEW_NOINLINE NLOHMANN_VIEW_NOCLONE inline c while (e - block >= 16) { const __m128i in = _mm_loadu_si128(static_cast(static_cast(block))); // NOLINT(bugprone-casting-through-void) - const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), - _mm_cmpeq_epi8(_mm_subs_epu8(in, _mm_set1_epi8(0x1F)), zero)); // in < 0x20 + // the quote and the control characters: (c ^ 2) < 33, that is, + // (c ^ 2) - 32 saturates to 0 (see vector_plain_run()) + const __m128i special = _mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('\\')), + _mm_cmpeq_epi8(_mm_subs_epu8(_mm_xor_si128(in, _mm_set1_epi8(2)), _mm_set1_epi8(32)), zero)); const __m128i prev1 = _mm_alignr_epi8(in, prev, 15); const __m128i sc = _mm_and_si128(_mm_and_si128(_mm_shuffle_epi8(t1h, _mm_and_si128(_mm_srli_epi16(prev1, 4), nibble)), _mm_shuffle_epi8(t1l, _mm_and_si128(prev1, nibble))), @@ -8600,9 +8607,12 @@ inline std::size_t plain_output_run(const unsigned char* s, std::size_t n) noexc for (; i + 16 <= n; i += 16) { const __m128i in = _mm_loadu_si128(static_cast(static_cast(s + i))); // NOLINT(bugprone-casting-through-void) - // (a byte is below 0x20 if it is its own minimum with 0x1F) - const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), - _mm_cmpeq_epi8(_mm_min_epu8(in, _mm_set1_epi8(0x1F)), in)); + // the quote and the control characters are the bytes with + // (c ^ 2) <= 32, that is, those that are their own minimum with 32 + // (see vector_plain_run()) + const __m128i flipped = _mm_xor_si128(in, _mm_set1_epi8(2)); + const __m128i special = _mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('\\')), + _mm_cmpeq_epi8(_mm_min_epu8(flipped, _mm_set1_epi8(32)), flipped)); const auto mask = static_cast(static_cast(_mm_movemask_epi8(special))); if (mask != 0) { @@ -8665,20 +8675,21 @@ inline bool is_ascii_without_delete(const char* s, std::size_t n) noexcept #if NLOHMANN_VIEW_VECTOR /// the bytes of the 16 at p that ensure_ascii cannot copy (a quote, a -/// backslash, a control character, DEL, or not ASCII; a signed compare with -/// 0x20 finds control characters and non-ASCII bytes at once), one bit per -/// byte (x86) or one nibble per byte (NEON) +/// backslash, a control character, DEL, or not ASCII; the signed compare +/// (int8_t)(c ^ 2) < 33 finds the quote, the control characters and the +/// non-ASCII bytes at once, see vector_plain_run()), one bit per byte (x86) or +/// one nibble per byte (NEON) NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t ascii_stop_mask(const unsigned char* p) noexcept { #if NLOHMANN_VIEW_NEON const uint8x16_t in = vld1q_u8(p); - const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('"')), vceqq_u8(in, vdupq_n_u8('\\'))), - vorrq_u8(vceqq_u8(in, vdupq_n_u8(0x7F)), vcltq_s8(vreinterpretq_s8_u8(in), vdupq_n_s8(0x20)))); + const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('\\')), vceqq_u8(in, vdupq_n_u8(0x7F))), + vcltq_s8(vreinterpretq_s8_u8(veorq_u8(in, vdupq_n_u8(2))), vdupq_n_s8(33))); return vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(special), 4)), 0); #else const __m128i in = _mm_loadu_si128(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) - const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), - _mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8(0x7F)), _mm_cmplt_epi8(in, _mm_set1_epi8(0x20)))); + const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('\\')), _mm_cmpeq_epi8(in, _mm_set1_epi8(0x7F))), + _mm_cmplt_epi8(_mm_xor_si128(in, _mm_set1_epi8(2)), _mm_set1_epi8(33))); return static_cast(static_cast(_mm_movemask_epi8(special))); #endif }