From 9fad5268882d27daaf5ccec738a096a135718d7b Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Wed, 28 Oct 2026 09:31:10 +0100 Subject: [PATCH] json_view: write the token digits without branches (GCC on x86-64) GCC 13 compiled the ?: selections of token_digits_1/2(), biased_distance() and the batch's classification loop as branches. Their conditions are as good as random, so dumping canada.json mispredicted about one branch per float (callgrind --branch-sim: 220k per dump with GCC, 97k with Clang), and the dump took 21-26 % longer than with Clang. The selections are now masks (mask_if(), select_if()), which no compiler turns into a branch. canada.json dump on the Haswell VPS (dprobe, best of 60, 3 interleaved rounds): GCC 13 4.74-5.04 ms -> 3.98-4.24 ms, Clang 18 unchanged (3.93-4.06 -> 3.82-3.98 ms); mispredictions per dump 220k -> 98k with GCC. Instructions per dump: GCC 38.2M -> 40.2M, Clang 34.7M -> 35.0M. Apple M1 Max, Apple clang 21: 0.5-1.8 % fewer cycles per dump in 4 of 4 pairs. In bench_all, json_view's dump of canada.json against yyjson goes from 0.75-0.78 to 0.91 with GCC. Output unchanged: dump() equals json::parse(text).dump() for 4e7 random 16/17-digit, %.17g and short tokens (singles and pairs) with GCC 13 on x86-64 and 6e7 with Apple clang on arm64; unit-json_view_dump passes with GCC 13 and with Clang under ASan/UBSan. Signed-off-by: Niels Lohmann --- include/nlohmann/detail/view/serializer.hpp | 40 ++++++++++++++------- single_include/nlohmann/json_view.hpp | 40 ++++++++++++++------- 2 files changed, 56 insertions(+), 24 deletions(-) diff --git a/include/nlohmann/detail/view/serializer.hpp b/include/nlohmann/detail/view/serializer.hpp index 69b137d47..2bfa2d2c0 100644 --- a/include/nlohmann/detail/view/serializer.hpp +++ b/include/nlohmann/detail/view/serializer.hpp @@ -447,11 +447,27 @@ NLOHMANN_VIEW_ALWAYS_INLINE int locate_token(std::uint64_t w, int q, std::uint64 return decided ? g : 3; } +/// all bits set if b, else none +NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t mask_if(bool b) noexcept +{ + return std::uint64_t{0} - static_cast(b); +} + +/// a if b, else c, without a branch (the conditions of token_digits_1/2() are +/// as good as random, and GCC compiled their ?: as branches: one misprediction +/// per float of canada.json on x86-64) +NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t select_if(bool b, std::uint64_t a, std::uint64_t c) noexcept +{ + return c ^ ((a ^ c) & mask_if(b)); +} + /// the distance of a position (see locate_token()) from the biased point 32 NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t biased_distance(std::uint64_t position) noexcept { constexpr std::uint64_t center = std::uint64_t{32} << 56u; - return position >= center ? position - center : center - position; + // |position - center|, without a branch: negated (complemented, plus 1) if below + const std::uint64_t below = mask_if(position < center); + return ((position - center) ^ below) - below; } /*! @@ -469,10 +485,10 @@ NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t token_digits_1(std::uint64_t w, std::u constexpr std::uint64_t one = std::uint64_t{1} << 56u; const std::uint64_t from = offset + (r10 * one); // x - (w - r10) (+ 32) const bool up = from > 37 * one; // (nearer to the multiple above) - const bool shorter = biased_distance(from - (up ? 10 * one : 0)) < radius; - const std::uint64_t a = w - r10 + (up ? 10u : 0u); + const bool shorter = biased_distance(from - (mask_if(up) & (10 * one))) < radius; + const std::uint64_t a = w - r10 + (mask_if(up) & 10u); const std::uint64_t b = w + ((offset + (one / 2)) >> 56u) - 32; // w rounded to x - return shorter ? a : b; + return select_if(shorter, a, b); } /*! @@ -485,15 +501,15 @@ NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t token_digits_2(std::uint64_t w, std::u constexpr std::uint64_t one = std::uint64_t{1} << 56u; const std::uint64_t from = offset + (r100 * one); const bool up = from > 82 * one; - const bool shorter = biased_distance(from - (up ? 100 * one : 0)) < radius; - const std::uint64_t a = w - r100 + (up ? 100u : 0u); + const bool shorter = biased_distance(from - (mask_if(up) & (100 * one))) < radius; + const std::uint64_t a = w - r100 + (mask_if(up) & 100u); // x - (w - r10), in [-12, 21]: the multiple of 10 nearest to x is that one, // or one or two above, or one below const std::uint64_t y = offset + (r10 * one); - const std::uint64_t steps = (y > 37 * one ? 1u : 0u) + (y > 47 * one ? 1u : 0u); + const std::uint64_t steps = static_cast(y > 37 * one) + static_cast(y > 47 * one); const bool down = y < 27 * one; - const std::uint64_t b = w - r10 + (10 * steps) - (down ? 10u : 0u); - return shorter ? a : b; + const std::uint64_t b = w - r10 + (10 * steps) - (mask_if(down) & 10u); + return select_if(shorter, a, b); } /// the four ASCII digits of x < 10^4, the first in the lowest byte (as @@ -1287,11 +1303,11 @@ class view_serializer // (the range excludes the values of bytes that are not digits, only // in a damaged image loaded with image_check::bounds) const int g = w - 1000000000000000u < 99000000000000000u ? locate_token(w, exponent[i], offset[i], radius[i]) : 3; - batch.digits[i] = g <= 0 ? (last_four[i] >> 16u) + 1 : 0; + batch.digits[i] = mask_if(g <= 0) & ((last_four[i] >> 16u) + 1); kind_1[count_1] = static_cast(i); - count_1 += g == 1 ? 1 : 0; + count_1 += static_cast(g == 1); kind_2[count_2] = static_cast(i); - count_2 += g == 2 ? 1 : 0; + count_2 += static_cast(g == 2); } // the digits to write, in one loop per kind (see mark()) for (std::size_t k = 0; k != count_1; ++k) diff --git a/single_include/nlohmann/json_view.hpp b/single_include/nlohmann/json_view.hpp index 284fcc947..c7eca0ddb 100644 --- a/single_include/nlohmann/json_view.hpp +++ b/single_include/nlohmann/json_view.hpp @@ -7026,11 +7026,27 @@ NLOHMANN_VIEW_ALWAYS_INLINE int locate_token(std::uint64_t w, int q, std::uint64 return decided ? g : 3; } +/// all bits set if b, else none +NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t mask_if(bool b) noexcept +{ + return std::uint64_t{0} - static_cast(b); +} + +/// a if b, else c, without a branch (the conditions of token_digits_1/2() are +/// as good as random, and GCC compiled their ?: as branches: one misprediction +/// per float of canada.json on x86-64) +NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t select_if(bool b, std::uint64_t a, std::uint64_t c) noexcept +{ + return c ^ ((a ^ c) & mask_if(b)); +} + /// the distance of a position (see locate_token()) from the biased point 32 NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t biased_distance(std::uint64_t position) noexcept { constexpr std::uint64_t center = std::uint64_t{32} << 56u; - return position >= center ? position - center : center - position; + // |position - center|, without a branch: negated (complemented, plus 1) if below + const std::uint64_t below = mask_if(position < center); + return ((position - center) ^ below) - below; } /*! @@ -7048,10 +7064,10 @@ NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t token_digits_1(std::uint64_t w, std::u constexpr std::uint64_t one = std::uint64_t{1} << 56u; const std::uint64_t from = offset + (r10 * one); // x - (w - r10) (+ 32) const bool up = from > 37 * one; // (nearer to the multiple above) - const bool shorter = biased_distance(from - (up ? 10 * one : 0)) < radius; - const std::uint64_t a = w - r10 + (up ? 10u : 0u); + const bool shorter = biased_distance(from - (mask_if(up) & (10 * one))) < radius; + const std::uint64_t a = w - r10 + (mask_if(up) & 10u); const std::uint64_t b = w + ((offset + (one / 2)) >> 56u) - 32; // w rounded to x - return shorter ? a : b; + return select_if(shorter, a, b); } /*! @@ -7064,15 +7080,15 @@ NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t token_digits_2(std::uint64_t w, std::u constexpr std::uint64_t one = std::uint64_t{1} << 56u; const std::uint64_t from = offset + (r100 * one); const bool up = from > 82 * one; - const bool shorter = biased_distance(from - (up ? 100 * one : 0)) < radius; - const std::uint64_t a = w - r100 + (up ? 100u : 0u); + const bool shorter = biased_distance(from - (mask_if(up) & (100 * one))) < radius; + const std::uint64_t a = w - r100 + (mask_if(up) & 100u); // x - (w - r10), in [-12, 21]: the multiple of 10 nearest to x is that one, // or one or two above, or one below const std::uint64_t y = offset + (r10 * one); - const std::uint64_t steps = (y > 37 * one ? 1u : 0u) + (y > 47 * one ? 1u : 0u); + const std::uint64_t steps = static_cast(y > 37 * one) + static_cast(y > 47 * one); const bool down = y < 27 * one; - const std::uint64_t b = w - r10 + (10 * steps) - (down ? 10u : 0u); - return shorter ? a : b; + const std::uint64_t b = w - r10 + (10 * steps) - (mask_if(down) & 10u); + return select_if(shorter, a, b); } /// the four ASCII digits of x < 10^4, the first in the lowest byte (as @@ -7866,11 +7882,11 @@ class view_serializer // (the range excludes the values of bytes that are not digits, only // in a damaged image loaded with image_check::bounds) const int g = w - 1000000000000000u < 99000000000000000u ? locate_token(w, exponent[i], offset[i], radius[i]) : 3; - batch.digits[i] = g <= 0 ? (last_four[i] >> 16u) + 1 : 0; + batch.digits[i] = mask_if(g <= 0) & ((last_four[i] >> 16u) + 1); kind_1[count_1] = static_cast(i); - count_1 += g == 1 ? 1 : 0; + count_1 += static_cast(g == 1); kind_2[count_2] = static_cast(i); - count_2 += g == 2 ? 1 : 0; + count_2 += static_cast(g == 2); } // the digits to write, in one loop per kind (see mark()) for (std::size_t k = 0; k != count_1; ++k)