mirror of
https://github.com/nlohmann/json.git
synced 2026-10-25 15:28:54 +00:00
json_view: dump canada.json faster than yyjson (batched float decisions)
Dump of float tokens of 16 or 17 digits without an exponent (SIMD targets, little-endian), in batches of up to 256 floats ahead of the walk, one loop per step: - the digits (16 bytes at a time) and the token's last four digits; - locate_token(): Eisel-Lemire's product gives the double x, and x's position relative to the token is exact in a 64-bit fixed-point number (56 bits after the point of the token's last digit), with the radius of its interval; - token_digits_1/2(), one loop per kind (g): the multiple of 10^g nearest to x if it is in the interval, else the multiple of 10^(g - 1) nearest to x, the digits that dump() (Zmij) writes. Exact ties between two candidates (like 754570942.33984373, whose double is ...33984375) are left to the full conversion; - the output is the token with its last four digits replaced; a rounding that carries further goes to the writer of Zmij's results. The batch storage is on the heap (allocated by the first conversion), and a pair of floats like [x, y] in a read-only document is written without a frame. Reading: get<double>() converts such tokens with a lean Eisel-Lemire (coordinate_float()). Output is byte-identical to the previous writer: 10^9 generated tokens in one array and 3 * 10^8 in arrays of 1 to 9 elements on Apple M1, 2.5 * 10^8 and 10^8 on x86-64 (all exponents, subnormals, 15-18 digits, exact binary fractions and ties, near-short decimals); get<double>() equals json::parse on 5 * 10^7 values. compare.py --download, Apple M1 Max, clang 21, 30 rounds, two runs (and two with aligned code: -falign-loops=64 -mllvm -align-all-functions=6); json_view, and the factor of yyjson 0.13.0 over it: - canada dump 3636 -> 1623 us (0.47x -> 1.06x; aligned 1.05x) - canada traverse 2287 -> 1817 us (0.85x -> 1.07x; aligned 0.99x) - canada patch 4628 -> 2735 us (0.80x -> 1.36x) - canada update 4763 -> 2911 us (0.78x -> 1.28x) Linux x86-64 (Haswell VPS, GCC 13, one run): canada dump 0.53x -> 0.72x, traverse 0.89x -> 0.95x, patch 1.01x -> 1.17x, update 0.96x -> 1.13x. Open: canada traverse in the aligned build (0.99x) and canada dump on x86-64 (0.72x) are still behind yyjson. Canada parse and select are about 2 % slower in bench_view in both builds, with identical parser code (likely layout; the corpus parse row is not slower). In the plain build only, rpc, status, and twitter dump are 2-4 % slower (0-6 % faster in the aligned build). Signed-off-by: Niels Lohmann <mail@nlohmann.me>
This commit is contained in:
4 files changed
+721
-266
No files matched your search
@@ -259,6 +259,80 @@ NLOHMANN_VIEW_NOINLINE FloatType edited_float(const char* token, const node& n)
|
||||
return float_value<FloatType>(token, n);
|
||||
}
|
||||
|
||||
#if NLOHMANN_VIEW_VECTOR
|
||||
/// eisel_lemire() for the token w * 10^q of coordinate_float()
|
||||
template<typename Format>
|
||||
NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t coordinate_bits(std::int64_t q, std::uint64_t w, Format /*format*/) noexcept
|
||||
{
|
||||
return eisel_lemire<Format>(q, w);
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief (binary64) eisel_lemire() without the cases that tokens of 16 or 17
|
||||
digits with 5 or more after the point cannot reach: a value of 0, a
|
||||
subnormal, an infinity, or a tie (q < -4)
|
||||
*/
|
||||
NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t coordinate_bits(std::int64_t q, std::uint64_t w, ieee_binary_format<53> format) noexcept
|
||||
{
|
||||
if (NLOHMANN_VIEW_UNLIKELY(w == 0 || q > -5))
|
||||
{
|
||||
return eisel_lemire<ieee_binary_format<53>>(q, w);
|
||||
}
|
||||
static_cast<void>(format);
|
||||
const int lz = count_leading_zeros(w);
|
||||
const std::uint64_t normalized = w << static_cast<unsigned>(lz);
|
||||
const auto index = static_cast<std::size_t>(2 * (q - pow5_128_smallest_power));
|
||||
const uint128_parts product = full_multiplication(normalized, pow5_128()[index]);
|
||||
std::uint64_t high = product.high;
|
||||
if (NLOHMANN_VIEW_UNLIKELY((high & 0x1FFu) == 0x1FFu))
|
||||
{
|
||||
// (the lower bits may carry into the result)
|
||||
const std::uint64_t second = full_multiplication(normalized, pow5_128()[index + 1]).high;
|
||||
high += product.low + second < product.low ? 1u : 0u;
|
||||
}
|
||||
const auto upperbit = static_cast<unsigned>(high >> 63u);
|
||||
std::uint64_t mantissa = high >> (upperbit + 9);
|
||||
std::uint64_t power2 = static_cast<std::uint64_t>(((217706 * q) >> 16) + 63 + static_cast<std::int64_t>(upperbit) - lz + 1023);
|
||||
mantissa = (mantissa + (mantissa & 1u)) >> 1u;
|
||||
const bool carry = mantissa >= (std::uint64_t{2} << 52u);
|
||||
mantissa = carry ? (std::uint64_t{1} << 52u) : mantissa;
|
||||
power2 += carry ? 1u : 0u;
|
||||
return (mantissa & ~(std::uint64_t{1} << 52u)) | (power2 << 52u);
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief (binary32 and binary64) the value of a float token of the source with
|
||||
16 or 17 digits, a point, and no exponent, the shape of most coordinates
|
||||
written with full precision; whether the token of n has this shape
|
||||
|
||||
These tokens skip the general layout of layout_decimal(): no digits beyond
|
||||
the 17th, no exponent (and Eisel-Lemire directly, as in layout_float()).
|
||||
*/
|
||||
template<typename FloatType>
|
||||
NLOHMANN_VIEW_ALWAYS_INLINE bool coordinate_float(const document_data& d, const node& n, FloatType& value) noexcept
|
||||
{
|
||||
const unsigned int_digits = n.extra & 0xFFu;
|
||||
const unsigned frac_digits = n.extra >> 8u;
|
||||
const unsigned count = int_digits + frac_digits;
|
||||
const auto* const first = reinterpret_cast<const unsigned char*>(d.src + n.off); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast)
|
||||
const unsigned negative = first[0] == '-' ? 1u : 0u;
|
||||
if (count - 16u > 1u || frac_digits == 0 || n.len != negative + count + 1 || d.size - n.off < 33)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
const unsigned char* const digits = first + negative;
|
||||
const std::uint64_t w = sixteen_digits(digits, int_digits);
|
||||
// (the 17th digit follows the point)
|
||||
const std::uint64_t w17 = (w * 10) + static_cast<std::uint64_t>(digits[17] - '0');
|
||||
using format = ieee_binary_format<std::numeric_limits<FloatType>::digits>;
|
||||
native_float_t<FloatType> result{};
|
||||
float_from_bits(coordinate_bits(-static_cast<std::int64_t>(frac_digits), count == 17 ? w17 : w, format{}) | (static_cast<std::uint64_t>(negative) << format::sign_bit()), result);
|
||||
value = result;
|
||||
return true;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
/// the value of the float token of a node, as parse() converts it; floats and
|
||||
/// doubles with at most 19 digits are converted from the digit layout
|
||||
template<typename FloatType>
|
||||
@@ -274,6 +348,13 @@ FloatType float_value(const document_data& d, const node& n)
|
||||
template<typename FloatType>
|
||||
FloatType float_value(const document_data& d, const node& n, std::true_type /*binary32 or binary64*/)
|
||||
{
|
||||
#if NLOHMANN_VIEW_VECTOR
|
||||
FloatType value {};
|
||||
if (coordinate_float(d, n, value))
|
||||
{
|
||||
return value;
|
||||
}
|
||||
#endif
|
||||
const unsigned int_digits = n.extra & 0xFFu;
|
||||
const unsigned frac_digits = n.extra >> 8u;
|
||||
if (NLOHMANN_VIEW_LIKELY(int_digits + frac_digits <= 19)) // (255 marks "many")
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
#include <cstring> // memcpy, memset
|
||||
#include <functional> // less
|
||||
#include <limits> // numeric_limits
|
||||
#include <memory> // unique_ptr
|
||||
#include <type_traits> // integral_constant
|
||||
#include <vector> // vector
|
||||
|
||||
@@ -386,89 +387,124 @@ struct depth_counter<false>
|
||||
};
|
||||
|
||||
/*!
|
||||
@brief the digits of the double nearest to a token w * 10^q (with the given
|
||||
bits, from eisel_lemire()) as dump() writes them (in units of 10^q), for
|
||||
10^15 <= w < 10^17 and -17 <= q <= -1; 0 if they are not decided here
|
||||
@brief where the double x nearest to a token w * 10^q lies, for 10^15 <= w <
|
||||
10^17 and -17 <= q <= -1 (Eisel-Lemire, then a fixed-point position)
|
||||
|
||||
dump() writes the shortest decimal in the rounding interval of the double x
|
||||
(the closest one if there are several, see zmij::to_shortest()). Its digits
|
||||
are those of a multiple of 10^g of the token's last digit, with g = dec_exp +
|
||||
1 - q for x's decimal exponent dec_exp (at most 2 here): the multiple of 10^g
|
||||
nearest to x if it lies in the interval (at most one does), else the multiple
|
||||
of 10^(g - 1) nearest to x. For g <= 0, the token itself is in the interval.
|
||||
dump() writes the shortest decimal in the rounding interval of x (the closest
|
||||
one if there are several, see zmij::to_shortest()). Its digits are those of a
|
||||
multiple of 10^g of the token's last digit, with g = dec_exp + 1 - q for x's
|
||||
decimal exponent dec_exp: the multiple of 10^g nearest to x if it lies in the
|
||||
interval (at most one does), else the multiple of 10^(g - 1) nearest to x
|
||||
(see token_digits_1() and token_digits_2()). For g <= 0, the token itself is
|
||||
in the interval.
|
||||
|
||||
All of it is exact in a 64-bit fixed-point number with 56 bits after the point
|
||||
(of the token's last digit): x / 10^q = 2c * 5^-q / 2^t for x = c * 2^e2 and t
|
||||
= q + 1 - e2, with at most 56 bits after the point here, and |x / 10^q - w| is
|
||||
at most half the spacing of the doubles, below 12 units. So only the low 64
|
||||
bits of 2c * 5^-q are needed, one multiplication beyond Eisel-Lemire's. An
|
||||
exact tie between two candidates and a power of two (whose interval is not
|
||||
symmetric) are left to the full conversion.
|
||||
The position is exact in a 64-bit fixed-point number with 56 bits after the
|
||||
point (of the token's last digit): x / 10^q = 2c * 5^-q / 2^t for x = c * 2^e2
|
||||
and t = q + 1 - e2, with at most 56 bits after the point here, and |x / 10^q -
|
||||
w| is at most half the spacing of the doubles, below 12 units. So only the low
|
||||
64 bits of 2c * 5^-q are needed. c and e2 come from Eisel-Lemire's product
|
||||
directly (as eisel_lemire() rounds it: no tie is possible for q < -4, and its
|
||||
second product is added where it can carry into the result).
|
||||
|
||||
@param[out] offset x / 10^q - w + 32, with 56 bits after the point
|
||||
@param[out] radius half the spacing of the doubles at x, likewise (+ 1 if
|
||||
the interval is closed)
|
||||
@return g; 3 if the digits are not decided here (a power of two, whose
|
||||
interval is not symmetric, or out of range)
|
||||
*/
|
||||
NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t shortest_token_digits(std::uint64_t w, int q, std::uint64_t bits) noexcept
|
||||
NLOHMANN_VIEW_ALWAYS_INLINE int locate_token(std::uint64_t w, int q, std::uint64_t& offset, std::uint64_t& radius) noexcept
|
||||
{
|
||||
const std::uint64_t fraction = bits & ((std::uint64_t{1} << 52u) - 1);
|
||||
const std::uint64_t c = fraction | (std::uint64_t{1} << 52u);
|
||||
const int e2 = static_cast<int>(bits >> 52u) - 1075;
|
||||
const int lz = count_leading_zeros(w);
|
||||
const std::uint64_t normalized = w << static_cast<unsigned>(lz);
|
||||
const auto index = static_cast<std::size_t>(2 * (q - pow5_128_smallest_power));
|
||||
const uint128_parts product = full_multiplication(normalized, pow5_128()[index]);
|
||||
std::uint64_t high = product.high;
|
||||
if (NLOHMANN_VIEW_UNLIKELY((high & 0x1FFu) == 0x1FFu))
|
||||
{
|
||||
// (as eisel_lemire(): the lower bits may carry into the result)
|
||||
const std::uint64_t second = full_multiplication(normalized, pow5_128()[index + 1]).high;
|
||||
high += product.low + second < product.low ? 1u : 0u;
|
||||
}
|
||||
const auto upperbit = static_cast<unsigned>(high >> 63u);
|
||||
const std::uint64_t mantissa = high >> (upperbit + 9);
|
||||
const std::uint64_t c = (mantissa + (mantissa & 1u)) >> 1u;
|
||||
// e2 as eisel_lemire() computes the exponent (log2(10) ~ 217706 / 2^16)
|
||||
const int e2 = static_cast<int>((217706 * static_cast<std::int64_t>(q)) >> 16) + 63 + static_cast<int>(upperbit) - lz - 52;
|
||||
const int t = q + 1 - e2;
|
||||
const int g = ((e2 * 315653) >> 20) + 1 - q; // (dec_exp as in zmij::to_shortest(), for a regular x)
|
||||
if (NLOHMANN_VIEW_UNLIKELY(fraction == 0 || t < 1 || t > 56 || g > 2))
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
if (g <= 0)
|
||||
{
|
||||
return w;
|
||||
}
|
||||
const unsigned s = static_cast<unsigned>(56 - t);
|
||||
const unsigned minus_q = static_cast<unsigned>(-q);
|
||||
const std::uint64_t five = ::nlohmann::detail::dtoa_impl::powers_of_ten()[minus_q] >> minus_q; // 5^-q
|
||||
constexpr std::uint64_t one = std::uint64_t{1} << 56u;
|
||||
// positions are kept 32 units above their value, so that they are not
|
||||
// negative; x and the candidates are at most 112 units apart
|
||||
constexpr std::uint64_t bias = 32;
|
||||
const std::uint64_t x = (2 * c * five) << s; // x / 10^q modulo 256 units
|
||||
const std::uint64_t half = five << s; // half the spacing of the doubles (the interval's radius)
|
||||
const bool even = (c & 1u) == 0; // (the interval is closed)
|
||||
// (the choices are masks, not conditions, which compilers could make
|
||||
// branches that these data mispredict)
|
||||
const std::uint64_t g1 = 0 - static_cast<std::uint64_t>(g == 1);
|
||||
const std::uint64_t r10 = w - ((w / 10) * 10);
|
||||
const std::uint64_t r100 = w - ((w / 100) * 100);
|
||||
// the multiple of G = 10^g nearest to x: a0 or a neighbor
|
||||
const std::uint64_t grid = (10u & g1) | (100u & ~g1);
|
||||
const std::uint64_t r = (r10 & g1) | (r100 & ~g1);
|
||||
const std::uint64_t at = x - ((w - r - bias) << 56u); // x - a0, biased
|
||||
const std::uint64_t up = 0 - static_cast<std::uint64_t>(at > (bias * one) + (grid * (one / 2)));
|
||||
const std::uint64_t down = 0 - static_cast<std::uint64_t>(at + (grid * (one / 2)) < bias * one);
|
||||
const std::uint64_t step = (grid & up) - (grid & down);
|
||||
const std::uint64_t center = (bias + step) * one;
|
||||
const std::uint64_t below = 0 - static_cast<std::uint64_t>(at < center);
|
||||
const std::uint64_t dist = ((center - at) & below) | ((at - center) & ~below);
|
||||
const std::uint64_t shorter = 0 - static_cast<std::uint64_t>(dist < half + (even ? 1u : 0u));
|
||||
const std::uint64_t a = w - r + step;
|
||||
// else the multiple of 1 (g == 1) or 10 (g == 2) nearest to x
|
||||
const std::uint64_t rl = r10 & ~g1;
|
||||
const std::uint64_t pos = x - ((w - rl - bias) << 56u) + ((one / 2) & g1) + ((5 * one) & ~g1); // x - b0 + half a step, biased
|
||||
const std::uint64_t whole = pos >> 56u; // in [20, 59]
|
||||
const std::uint64_t m = whole + 20 - bias; // (a multiple of 10 above whole - bias)
|
||||
const std::uint64_t tens = (m * 205) >> 11u; // m / 10
|
||||
const std::uint64_t tie = 0 - static_cast<std::uint64_t>((pos & (one - 1)) == 0 && (g == 1 || m == tens * 10));
|
||||
const std::uint64_t b = ((w + whole - bias) & g1) | ((w - rl + (tens * 10) - 20) & ~g1);
|
||||
return (a & shorter) | (b & ~tie & ~shorter);
|
||||
static const std::array<std::uint64_t, 17> fives = {{1u, 5u, 25u, 125u, 625u, 3125u, 15625u, 78125u, 390625u, 1953125u, 9765625u, 48828125u, 244140625u, 1220703125u, 6103515625u, 30517578125u, 152587890625u}};
|
||||
const std::uint64_t five = fives[static_cast<std::size_t>(-q)]; // 5^-q
|
||||
const unsigned s = static_cast<unsigned>(56 - t) & 63u;
|
||||
offset = ((2 * c * five) << s) - (w << 56u) + (std::uint64_t{32} << 56u);
|
||||
radius = (five << s) + ((c & 1u) == 0 ? 1u : 0u);
|
||||
// (c = 2^52: a power of two; c = 2^53: rounded up to one. With 8 to 16
|
||||
// fraction digits, as the batch's tokens have, t is in [15, 40] and g at
|
||||
// most 2. x lies halfway between two candidates only if its distance from
|
||||
// the token is a whole unit (g == 2) or a half (g == 1): such tokens are
|
||||
// left to the full conversion)
|
||||
const bool decided = c - ((std::uint64_t{1} << 52u) + 1) < (std::uint64_t{1} << 52u) - 1
|
||||
&& (offset & ((std::uint64_t{1} << 55u) - 1)) != 0;
|
||||
return decided ? g : 3;
|
||||
}
|
||||
|
||||
/// the eight ASCII digits of x < 10^8, the first in the lowest byte (as
|
||||
/// little-endian memory holds them)
|
||||
NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t eight_digit_text(std::uint64_t x) noexcept
|
||||
/// the distance of a position (see locate_token()) from the biased point 32
|
||||
NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t biased_distance(std::uint64_t position) noexcept
|
||||
{
|
||||
const std::uint64_t abcd = (x * 109951163u) >> 40u; // x / 10000
|
||||
std::uint64_t v = abcd | ((x - (abcd * 10000u)) << 32u);
|
||||
const std::uint64_t ab = ((v * 10486u) >> 20u) & 0x0000007F0000007Fu; // / 100 per 32-bit lane
|
||||
v = ab | ((v - (ab * 100u)) << 16u);
|
||||
const std::uint64_t a = ((v * 103u) >> 10u) & 0x000F000F000F000Fu; // / 10 per 16-bit lane
|
||||
constexpr std::uint64_t center = std::uint64_t{32} << 56u;
|
||||
return position >= center ? position - center : center - position;
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief (g == 1) the digits dump() writes for a token, from the position of its
|
||||
double (see locate_token()): the multiple of 10 nearest to x if it is in the
|
||||
interval, else w rounded to x (an exact tie between two candidates, like
|
||||
754570942.33984373, whose double is 754570942.33984375, is excluded by
|
||||
locate_token())
|
||||
|
||||
With |x - w| at most the radius, which is below half the spacing of the
|
||||
candidates, the multiple of 10 nearest to x is w - r10 or the one above it.
|
||||
*/
|
||||
NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t token_digits_1(std::uint64_t w, std::uint64_t offset, std::uint64_t radius, std::uint64_t r10) noexcept
|
||||
{
|
||||
constexpr std::uint64_t one = std::uint64_t{1} << 56u;
|
||||
const std::uint64_t from = offset + (r10 * one); // x - (w - r10) (+ 32)
|
||||
const bool up = from > 37 * one; // (nearer to the multiple above)
|
||||
const bool shorter = biased_distance(from - (up ? 10 * one : 0)) < radius;
|
||||
const std::uint64_t a = w - r10 + (up ? 10u : 0u);
|
||||
const std::uint64_t b = w + ((offset + (one / 2)) >> 56u) - 32; // w rounded to x
|
||||
return shorter ? a : b;
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief (g == 2) the digits dump() writes for a token, as token_digits_1(): the
|
||||
multiple of 100 nearest to x if it is in the interval, else the multiple of 10
|
||||
nearest to x
|
||||
*/
|
||||
NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t token_digits_2(std::uint64_t w, std::uint64_t offset, std::uint64_t radius, std::uint64_t r10, std::uint64_t r100) noexcept
|
||||
{
|
||||
constexpr std::uint64_t one = std::uint64_t{1} << 56u;
|
||||
const std::uint64_t from = offset + (r100 * one);
|
||||
const bool up = from > 82 * one;
|
||||
const bool shorter = biased_distance(from - (up ? 100 * one : 0)) < radius;
|
||||
const std::uint64_t a = w - r100 + (up ? 100u : 0u);
|
||||
// x - (w - r10), in [-12, 21]: the multiple of 10 nearest to x is that one,
|
||||
// or one or two above, or one below
|
||||
const std::uint64_t y = offset + (r10 * one);
|
||||
const std::uint64_t steps = (y > 37 * one ? 1u : 0u) + (y > 47 * one ? 1u : 0u);
|
||||
const bool down = y < 27 * one;
|
||||
const std::uint64_t b = w - r10 + (10 * steps) - (down ? 10u : 0u);
|
||||
return shorter ? a : b;
|
||||
}
|
||||
|
||||
/// the four ASCII digits of x < 10^4, the first in the lowest byte (as
|
||||
/// little-endian memory holds them)
|
||||
NLOHMANN_VIEW_ALWAYS_INLINE std::uint32_t four_digit_text(std::uint32_t x) noexcept
|
||||
{
|
||||
const std::uint32_t ab = (x * 5243u) >> 19u; // x / 100
|
||||
std::uint32_t v = ab | ((x - (ab * 100u)) << 16u);
|
||||
const std::uint32_t a = ((v * 103u) >> 10u) & 0x000F000Fu; // / 10 per 16-bit lane
|
||||
v = a | ((v - (a * 10u)) << 8u);
|
||||
return v + 0x3030303030303030u;
|
||||
return v + 0x30303030u;
|
||||
}
|
||||
|
||||
/// how the view's dump() writes a value
|
||||
@@ -938,7 +974,7 @@ class view_serializer
|
||||
else if (enabled(fast_double::value))
|
||||
{
|
||||
room(64);
|
||||
w = write_double_ahead(w, n, batch);
|
||||
w = write_double_ahead(w, src, n, batch);
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -963,6 +999,20 @@ class view_serializer
|
||||
case value_t::array:
|
||||
{
|
||||
const bool object = n->kind == static_cast<std::uint8_t>(value_t::object);
|
||||
// (read-only documents, compact output) a pair of floats,
|
||||
// like the coordinates of GeoJSON, without a frame
|
||||
if (!Editable && !Pretty && !SourceNumbers && enabled(fast_double::value) && n->next == 3 && n->len == 2
|
||||
&& n[1].kind == static_cast<std::uint8_t>(value_t::number_float) && n[2].kind == static_cast<std::uint8_t>(value_t::number_float))
|
||||
{
|
||||
room(2 * 64);
|
||||
*w++ = '[';
|
||||
w = write_double_ahead(w, src, n + 1, batch);
|
||||
*w++ = ',';
|
||||
w = write_double_ahead(w, src, n + 2, batch);
|
||||
*w++ = ']';
|
||||
n += 2; // (the element before the next node)
|
||||
break;
|
||||
}
|
||||
room(8);
|
||||
if (n->len == 0)
|
||||
{
|
||||
@@ -1072,13 +1122,31 @@ class view_serializer
|
||||
}
|
||||
}
|
||||
|
||||
/// (float_batch) the mark of digits whose rounding carried beyond the
|
||||
/// token's last four (see mark())
|
||||
static constexpr std::uint64_t carried() noexcept
|
||||
{
|
||||
return std::uint64_t{1} << 63u;
|
||||
}
|
||||
|
||||
/// floats of 16 or 17 digits ahead of the walk, converted together (see
|
||||
/// convert_ahead()); the digits are those of shortest_token_digits(), 0
|
||||
/// for a float the batch does not write
|
||||
/// convert_ahead()), with their digits as mark() keeps them
|
||||
struct float_batch
|
||||
{
|
||||
std::array<const node*, 32> at; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read
|
||||
std::array<std::uint64_t, 32> digits; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read
|
||||
static constexpr std::size_t capacity() noexcept
|
||||
{
|
||||
return 256;
|
||||
}
|
||||
/// (on the heap, allocated by the first conversion: a frame of more
|
||||
/// than a page made every dump of a small document slower)
|
||||
struct entries
|
||||
{
|
||||
std::array<const node*, 256> at; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read
|
||||
std::array<std::uint64_t, 256> digits; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read
|
||||
};
|
||||
std::unique_ptr<entries> storage{};
|
||||
const node** at = nullptr;
|
||||
std::uint64_t* digits = nullptr;
|
||||
std::size_t count = 0;
|
||||
std::size_t next = 0;
|
||||
const node* scanned = nullptr; ///< where the last scan of the node array ended
|
||||
@@ -1086,21 +1154,24 @@ class view_serializer
|
||||
|
||||
/// (doubles) the float at n as dump() writes it, at w (64 bytes of room):
|
||||
/// from the batch if it holds n
|
||||
NLOHMANN_VIEW_ALWAYS_INLINE char* write_double_ahead(char* w, const node* n, float_batch& batch)
|
||||
NLOHMANN_VIEW_ALWAYS_INLINE char* write_double_ahead(char* w, const char* src, const node* n, float_batch& batch)
|
||||
{
|
||||
#if NLOHMANN_VIEW_NEON
|
||||
if (batch.next != batch.count && batch.at[batch.next] == n && batch.digits[batch.next] != 0)
|
||||
#if NLOHMANN_VIEW_VECTOR && NLOHMANN_VIEW_LITTLE_ENDIAN
|
||||
// (digits 0: not written by the batch; carried(): rounding carried
|
||||
// beyond the last four digits)
|
||||
if (batch.next != batch.count && batch.at[batch.next] == n && batch.digits[batch.next] - 1 < carried() - 1)
|
||||
{
|
||||
return write_token_digits(w, m_doc.src + n->off, n->len, n->extra >> 8u, batch.digits[batch.next++]);
|
||||
return write_token_digits(w, src, *n, batch.digits[batch.next++]);
|
||||
}
|
||||
return write_double_beside(w, n, batch);
|
||||
#else
|
||||
static_cast<void>(src);
|
||||
static_cast<void>(batch);
|
||||
return write_double_at(w, *n);
|
||||
#endif
|
||||
}
|
||||
|
||||
#if NLOHMANN_VIEW_NEON
|
||||
#if NLOHMANN_VIEW_VECTOR && NLOHMANN_VIEW_LITTLE_ENDIAN
|
||||
/// write_double_ahead() for a float that is not the batch's next one (out
|
||||
/// of line: the walk is faster with less code in its loop)
|
||||
NLOHMANN_VIEW_NOINLINE char* write_double_beside(char* w, const node* n, float_batch& batch)
|
||||
@@ -1114,9 +1185,13 @@ class view_serializer
|
||||
if ((batch.next != batch.count && batch.at[batch.next] == n) || convert_ahead(n, batch))
|
||||
{
|
||||
const std::uint64_t digits = batch.digits[batch.next++];
|
||||
if ((digits & carried()) != 0)
|
||||
{
|
||||
return write_carried_digits(w, *n, digits & ~carried());
|
||||
}
|
||||
if (digits != 0)
|
||||
{
|
||||
return write_token_digits(w, m_doc.src + n->off, n->len, n->extra >> 8u, digits);
|
||||
return write_token_digits(w, m_doc.src, *n, digits);
|
||||
}
|
||||
}
|
||||
return write_double_at(w, *n);
|
||||
@@ -1127,16 +1202,14 @@ class view_serializer
|
||||
array at once; whether n is the first of them
|
||||
|
||||
Such tokens (like the coordinates of GeoJSON, written with 17 digits) are
|
||||
written as the token with its last digits replaced (write_token_digits()).
|
||||
Each step (the digits, Eisel-Lemire, shortest_token_digits()) is a long
|
||||
chain of dependent multiplications; one loop per step over many floats lets
|
||||
the processor overlap the floats, which it does not across the steps of
|
||||
one float (on Apple M1, the dump of canada.json takes 30 % less time than
|
||||
with a conversion of each float, of which 10 points are this). Each node of the array
|
||||
written as the token with its last four digits replaced
|
||||
(write_token_digits()). Each step (the digits, locate_token(),
|
||||
token_digits_1/2()) is a long chain of dependent operations; one loop per
|
||||
step over many floats lets the processor overlap the floats, which it does
|
||||
not across the steps of one float (on Apple M1, one loop for the digits and
|
||||
locate_token() made the dump of canada.json 12 % slower). Each node of the array
|
||||
is scanned once; a node that the walk does not reach in order (in an
|
||||
editable document) is written without the batch. (AArch64 only: on x86-64
|
||||
(Haswell, GCC 13 and Clang 18), the batch was not faster than the
|
||||
conversion of each float.)
|
||||
editable document) is written without the batch.
|
||||
*/
|
||||
NLOHMANN_VIEW_NOINLINE bool convert_ahead(const node* n, float_batch& batch)
|
||||
{
|
||||
@@ -1148,72 +1221,144 @@ class view_serializer
|
||||
}
|
||||
const char* const src = m_doc.src;
|
||||
const char* const src_end = src + m_doc.size;
|
||||
const node* const stop = last - n > 128 ? n + 128 : last;
|
||||
const node* const stop = last - n > 1024 ? n + 1024 : last;
|
||||
batch.count = 0;
|
||||
batch.next = 0;
|
||||
// the digits of each token (and its last four from the text)
|
||||
std::array<std::uint64_t, 256> digits; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read
|
||||
std::array<int, 256> exponent; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read
|
||||
std::array<std::uint32_t, 256> last_four; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read
|
||||
const node* p = n;
|
||||
for (; p != stop && batch.count != batch.at.size(); ++p)
|
||||
if (!batch.storage)
|
||||
{
|
||||
batch.storage.reset(new typename float_batch::entries); // NOLINT(cppcoreguidelines-owning-memory,modernize-make-unique): C++11
|
||||
batch.at = batch.storage->at.data();
|
||||
batch.digits = batch.storage->digits.data();
|
||||
}
|
||||
for (; p != stop && batch.count != float_batch::capacity(); ++p)
|
||||
{
|
||||
// a token of the source: 16 or 17 digits, at least 8 after the
|
||||
// point, no exponent, no leading zero, 32 bytes to read
|
||||
// point, no exponent, 32 bytes to read
|
||||
const unsigned frac_digits = p->extra >> 8u;
|
||||
const unsigned count = (p->extra & 0xFFu) + frac_digits;
|
||||
const unsigned int_digits = p->extra & 0xFFu;
|
||||
const unsigned count = int_digits + frac_digits;
|
||||
if (p->kind == static_cast<std::uint8_t>(value_t::number_float) && (p->flags & node_flags::storage) == 0
|
||||
&& count - 16u <= 1u && frac_digits >= 8 && src_end - (src + p->off) >= 33)
|
||||
{
|
||||
const char* const token = src + p->off;
|
||||
const auto* const token = reinterpret_cast<const unsigned char*>(src + p->off); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast)
|
||||
const unsigned sign = token[0] == '-' ? 1 : 0;
|
||||
if (p->len == sign + count + 1 && token[sign] != '0')
|
||||
{
|
||||
batch.at[batch.count++] = p;
|
||||
}
|
||||
const unsigned char* const first = token + sign;
|
||||
const std::uint64_t w = sixteen_digits(first, int_digits);
|
||||
// (the 17th digit follows the point; a token with an exponent
|
||||
// gets digits outside the range of locate_token(), and is not
|
||||
// written by the batch. A token 0.xxx with 16 significant
|
||||
// digits is: its double is written as "0." and its digits too)
|
||||
const std::uint64_t w17 = (w * 10) + static_cast<std::uint64_t>(first[17] - '0');
|
||||
const std::size_t i = batch.count++;
|
||||
batch.at[i] = p;
|
||||
digits[i] = p->len != sign + count + 1 ? 0 : (count == 17 ? w17 : w);
|
||||
exponent[i] = -static_cast<int>(frac_digits);
|
||||
// (byte 0: the last digit, byte 1: the last two, bits 16-29: the last four)
|
||||
std::uint32_t four = 0;
|
||||
std::memcpy(&four, token + p->len - 4, 4);
|
||||
four -= 0x30303030u;
|
||||
const std::uint32_t pairs = ((four & 0x0F0F0F0Fu) * 2561u) >> 8u; // (byte 0, 2: the pairs)
|
||||
last_four[i] = (four >> 24u) | ((pairs >> 16u & 0xFFu) << 8u) | ((((pairs & 0x00FF00FFu) * 6553601u) >> 16u) << 16u);
|
||||
}
|
||||
}
|
||||
batch.scanned = p;
|
||||
// the position of each double, and the floats of either kind
|
||||
std::array<std::uint64_t, 256> offset; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read
|
||||
std::array<std::uint64_t, 256> radius; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read
|
||||
std::array<std::uint8_t, 256> kind_1; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read
|
||||
std::array<std::uint8_t, 256> kind_2; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read
|
||||
std::size_t count_1 = 0;
|
||||
std::size_t count_2 = 0;
|
||||
for (std::size_t i = 0; i != batch.count; ++i)
|
||||
{
|
||||
const node& f = *batch.at[i];
|
||||
const auto* digits = reinterpret_cast<const unsigned char*>(src + f.off); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast)
|
||||
digits += *digits == '-' ? 1 : 0;
|
||||
const std::uint64_t w = sixteen_digits(digits, f.extra & 0xFFu);
|
||||
// (the 17th digit follows the point)
|
||||
batch.digits[i] = (f.extra & 0xFFu) + (f.extra >> 8u) == 17 ? (w * 10) + static_cast<std::uint64_t>(digits[17] - '0') : w;
|
||||
const std::uint64_t w = digits[i];
|
||||
// (the range excludes the values of bytes that are not digits, only
|
||||
// in a damaged image loaded with image_check::bounds)
|
||||
const int g = w - 1000000000000000u < 99000000000000000u ? locate_token(w, exponent[i], offset[i], radius[i]) : 3;
|
||||
batch.digits[i] = g <= 0 ? (last_four[i] >> 16u) + 1 : 0;
|
||||
kind_1[count_1] = static_cast<std::uint8_t>(i);
|
||||
count_1 += g == 1 ? 1 : 0;
|
||||
kind_2[count_2] = static_cast<std::uint8_t>(i);
|
||||
count_2 += g == 2 ? 1 : 0;
|
||||
}
|
||||
std::array<std::uint64_t, 32> bits; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read
|
||||
for (std::size_t i = 0; i != batch.count; ++i)
|
||||
// the digits to write, in one loop per kind (see mark())
|
||||
for (std::size_t k = 0; k != count_1; ++k)
|
||||
{
|
||||
bits[i] = eisel_lemire<ieee_binary_format<53>>(-static_cast<int>(batch.at[i]->extra >> 8u), batch.digits[i]);
|
||||
const std::size_t i = kind_1[k];
|
||||
const std::uint64_t s = token_digits_1(digits[i], offset[i], radius[i], last_four[i] & 0xFFu);
|
||||
batch.digits[i] = mark(digits[i], s, last_four[i] >> 16u);
|
||||
}
|
||||
for (std::size_t i = 0; i != batch.count; ++i)
|
||||
for (std::size_t k = 0; k != count_2; ++k)
|
||||
{
|
||||
const std::uint64_t w = batch.digits[i];
|
||||
// (the range also excludes the values of bytes that are not
|
||||
// digits, only in a damaged image loaded with image_check::bounds)
|
||||
const std::uint64_t s = w - 1000000000000000u < 99000000000000000u ? shortest_token_digits(w, -static_cast<int>(batch.at[i]->extra >> 8u), bits[i]) : 0;
|
||||
// (only the last eight digits are written over the token's)
|
||||
batch.digits[i] = s / 100000000u == w / 100000000u ? s : 0;
|
||||
const std::size_t i = kind_2[k];
|
||||
const std::uint64_t s = token_digits_2(digits[i], offset[i], radius[i], last_four[i] & 0xFFu, (last_four[i] >> 8u) & 0xFFu);
|
||||
batch.digits[i] = mark(digits[i], s, last_four[i] >> 16u);
|
||||
}
|
||||
return batch.count != 0 && batch.at[0] == n;
|
||||
}
|
||||
|
||||
/// a float token of 16 or 17 digits as dump() writes it, at w (64 bytes
|
||||
/// of room): the token with its last eight digits replaced by those of
|
||||
/// digits (see shortest_token_digits(); the others are the token's), and
|
||||
/// without trailing zeros after the point, but one
|
||||
static char* write_token_digits(char* w, const char* token, std::uint32_t len, unsigned frac_digits, std::uint64_t digits) noexcept
|
||||
/*!
|
||||
@brief a float token of 16 or 17 digits as dump() writes it, at w (64 bytes
|
||||
of room), from the digits of its double (see locate_token()) whose
|
||||
rounding carried beyond the token's last four digits
|
||||
|
||||
(like 43.42027299999999 -> 43.420273): the digits go to the writer of
|
||||
Zmij's results as its 15 or 16 leading digits and one more, which omits
|
||||
the trailing zeros.
|
||||
*/
|
||||
NLOHMANN_VIEW_NOINLINE char* write_carried_digits(char* w, const node& n, std::uint64_t digits) const noexcept
|
||||
{
|
||||
const int exponent = -static_cast<int>(n.extra >> 8u);
|
||||
const std::uint64_t leading = digits / 10;
|
||||
const auto last = static_cast<unsigned char>(digits - (leading * 10));
|
||||
const bool seventeen = digits >= 10000000000000000u;
|
||||
*w = '-';
|
||||
w += m_doc.src[n.off] == '-' ? 1 : 0;
|
||||
return ::nlohmann::detail::dtoa_impl::write_shortest(w, seventeen ? ::nlohmann::detail::zmij::shortest_decimal{leading, exponent, last, last != 0}
|
||||
: ::nlohmann::detail::zmij::shortest_decimal{digits, exponent - 1, 0, false});
|
||||
}
|
||||
|
||||
/// the digits s of a token with the digits w and the last four digits
|
||||
/// r4, as the batch keeps them: the new last four digits + 1 (only those
|
||||
/// are written over the token's), or s marked if it differs from w beyond
|
||||
/// them (0 stays for a float the batch does not write)
|
||||
static std::uint64_t mark(std::uint64_t w, std::uint64_t s, std::uint64_t r4) noexcept
|
||||
{
|
||||
const std::uint64_t last = s - (w - r4);
|
||||
return last < 10000 ? last + 1 : s | carried();
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief a float token of 16 or 17 digits as dump() writes it, at w (64 bytes
|
||||
of room): the token with its last four digits replaced by digits - 1 (see
|
||||
mark(); the others are the token's), and without trailing zeros after the
|
||||
point, but one
|
||||
*/
|
||||
char* write_token_digits(char* w, const char* src, const node& n, std::uint64_t digits)
|
||||
{
|
||||
const char* const token = src + n.off;
|
||||
std::memcpy(w, token, 32);
|
||||
char* const end = w + len;
|
||||
const std::uint64_t last = eight_digit_text(digits % 100000000u);
|
||||
std::memcpy(end - 8, &last, 8);
|
||||
char* const end = w + n.len;
|
||||
const std::uint32_t last = four_digit_text(static_cast<std::uint32_t>(digits - 1));
|
||||
std::memcpy(end - 4, &last, 4);
|
||||
// (the token's text before the new digits, from the source: reading
|
||||
// the output back would wait for the stores)
|
||||
std::uint64_t before = 0;
|
||||
std::memcpy(&before, token + len - 16, 8);
|
||||
const std::uint64_t last_zeros = last ^ 0x3030303030303030u;
|
||||
std::memcpy(&before, token + n.len - 12, 8);
|
||||
const std::uint32_t last_zeros = last ^ 0x30303030u;
|
||||
const std::uint64_t before_zeros = before ^ 0x3030303030303030u;
|
||||
const unsigned zeros = last_zeros != 0 ? static_cast<unsigned>(count_leading_zeros(last_zeros)) / 8
|
||||
: 8 + (before_zeros != 0 ? static_cast<unsigned>(count_leading_zeros(before_zeros)) / 8 : 8);
|
||||
if (NLOHMANN_VIEW_UNLIKELY(last_zeros == 0 && before_zeros == 0))
|
||||
{
|
||||
return write_double_at(w, n); // (twelve or more trailing zeros)
|
||||
}
|
||||
const unsigned zeros = last_zeros != 0 ? static_cast<unsigned>(count_leading_zeros(last_zeros)) / 8 - 4
|
||||
: 4 + (static_cast<unsigned>(count_leading_zeros(before_zeros)) / 8);
|
||||
const unsigned frac_digits = n.extra >> 8u;
|
||||
return end - (zeros < frac_digits ? zeros : frac_digits - 1);
|
||||
}
|
||||
#endif
|
||||
|
||||
Generated
+356
-130
@@ -4846,6 +4846,80 @@ NLOHMANN_VIEW_NOINLINE FloatType edited_float(const char* token, const node& n)
|
||||
return float_value<FloatType>(token, n);
|
||||
}
|
||||
|
||||
#if NLOHMANN_VIEW_VECTOR
|
||||
/// eisel_lemire() for the token w * 10^q of coordinate_float()
|
||||
template<typename Format>
|
||||
NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t coordinate_bits(std::int64_t q, std::uint64_t w, Format /*format*/) noexcept
|
||||
{
|
||||
return eisel_lemire<Format>(q, w);
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief (binary64) eisel_lemire() without the cases that tokens of 16 or 17
|
||||
digits with 5 or more after the point cannot reach: a value of 0, a
|
||||
subnormal, an infinity, or a tie (q < -4)
|
||||
*/
|
||||
NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t coordinate_bits(std::int64_t q, std::uint64_t w, ieee_binary_format<53> format) noexcept
|
||||
{
|
||||
if (NLOHMANN_VIEW_UNLIKELY(w == 0 || q > -5))
|
||||
{
|
||||
return eisel_lemire<ieee_binary_format<53>>(q, w);
|
||||
}
|
||||
static_cast<void>(format);
|
||||
const int lz = count_leading_zeros(w);
|
||||
const std::uint64_t normalized = w << static_cast<unsigned>(lz);
|
||||
const auto index = static_cast<std::size_t>(2 * (q - pow5_128_smallest_power));
|
||||
const uint128_parts product = full_multiplication(normalized, pow5_128()[index]);
|
||||
std::uint64_t high = product.high;
|
||||
if (NLOHMANN_VIEW_UNLIKELY((high & 0x1FFu) == 0x1FFu))
|
||||
{
|
||||
// (the lower bits may carry into the result)
|
||||
const std::uint64_t second = full_multiplication(normalized, pow5_128()[index + 1]).high;
|
||||
high += product.low + second < product.low ? 1u : 0u;
|
||||
}
|
||||
const auto upperbit = static_cast<unsigned>(high >> 63u);
|
||||
std::uint64_t mantissa = high >> (upperbit + 9);
|
||||
std::uint64_t power2 = static_cast<std::uint64_t>(((217706 * q) >> 16) + 63 + static_cast<std::int64_t>(upperbit) - lz + 1023);
|
||||
mantissa = (mantissa + (mantissa & 1u)) >> 1u;
|
||||
const bool carry = mantissa >= (std::uint64_t{2} << 52u);
|
||||
mantissa = carry ? (std::uint64_t{1} << 52u) : mantissa;
|
||||
power2 += carry ? 1u : 0u;
|
||||
return (mantissa & ~(std::uint64_t{1} << 52u)) | (power2 << 52u);
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief (binary32 and binary64) the value of a float token of the source with
|
||||
16 or 17 digits, a point, and no exponent, the shape of most coordinates
|
||||
written with full precision; whether the token of n has this shape
|
||||
|
||||
These tokens skip the general layout of layout_decimal(): no digits beyond
|
||||
the 17th, no exponent (and Eisel-Lemire directly, as in layout_float()).
|
||||
*/
|
||||
template<typename FloatType>
|
||||
NLOHMANN_VIEW_ALWAYS_INLINE bool coordinate_float(const document_data& d, const node& n, FloatType& value) noexcept
|
||||
{
|
||||
const unsigned int_digits = n.extra & 0xFFu;
|
||||
const unsigned frac_digits = n.extra >> 8u;
|
||||
const unsigned count = int_digits + frac_digits;
|
||||
const auto* const first = reinterpret_cast<const unsigned char*>(d.src + n.off); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast)
|
||||
const unsigned negative = first[0] == '-' ? 1u : 0u;
|
||||
if (count - 16u > 1u || frac_digits == 0 || n.len != negative + count + 1 || d.size - n.off < 33)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
const unsigned char* const digits = first + negative;
|
||||
const std::uint64_t w = sixteen_digits(digits, int_digits);
|
||||
// (the 17th digit follows the point)
|
||||
const std::uint64_t w17 = (w * 10) + static_cast<std::uint64_t>(digits[17] - '0');
|
||||
using format = ieee_binary_format<std::numeric_limits<FloatType>::digits>;
|
||||
native_float_t<FloatType> result{};
|
||||
float_from_bits(coordinate_bits(-static_cast<std::int64_t>(frac_digits), count == 17 ? w17 : w, format{}) | (static_cast<std::uint64_t>(negative) << format::sign_bit()), result);
|
||||
value = result;
|
||||
return true;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
/// the value of the float token of a node, as parse() converts it; floats and
|
||||
/// doubles with at most 19 digits are converted from the digit layout
|
||||
template<typename FloatType>
|
||||
@@ -4861,6 +4935,13 @@ FloatType float_value(const document_data& d, const node& n)
|
||||
template<typename FloatType>
|
||||
FloatType float_value(const document_data& d, const node& n, std::true_type /*binary32 or binary64*/)
|
||||
{
|
||||
#if NLOHMANN_VIEW_VECTOR
|
||||
FloatType value {};
|
||||
if (coordinate_float(d, n, value))
|
||||
{
|
||||
return value;
|
||||
}
|
||||
#endif
|
||||
const unsigned int_digits = n.extra & 0xFFu;
|
||||
const unsigned frac_digits = n.extra >> 8u;
|
||||
if (NLOHMANN_VIEW_LIKELY(int_digits + frac_digits <= 19)) // (255 marks "many")
|
||||
@@ -6434,6 +6515,7 @@ NLOHMANN_JSON_NAMESPACE_END
|
||||
#include <cstring> // memcpy, memset
|
||||
#include <functional> // less
|
||||
#include <limits> // numeric_limits
|
||||
#include <memory> // unique_ptr
|
||||
#include <type_traits> // integral_constant
|
||||
#include <vector> // vector
|
||||
|
||||
@@ -6809,89 +6891,124 @@ struct depth_counter<false>
|
||||
};
|
||||
|
||||
/*!
|
||||
@brief the digits of the double nearest to a token w * 10^q (with the given
|
||||
bits, from eisel_lemire()) as dump() writes them (in units of 10^q), for
|
||||
10^15 <= w < 10^17 and -17 <= q <= -1; 0 if they are not decided here
|
||||
@brief where the double x nearest to a token w * 10^q lies, for 10^15 <= w <
|
||||
10^17 and -17 <= q <= -1 (Eisel-Lemire, then a fixed-point position)
|
||||
|
||||
dump() writes the shortest decimal in the rounding interval of the double x
|
||||
(the closest one if there are several, see zmij::to_shortest()). Its digits
|
||||
are those of a multiple of 10^g of the token's last digit, with g = dec_exp +
|
||||
1 - q for x's decimal exponent dec_exp (at most 2 here): the multiple of 10^g
|
||||
nearest to x if it lies in the interval (at most one does), else the multiple
|
||||
of 10^(g - 1) nearest to x. For g <= 0, the token itself is in the interval.
|
||||
dump() writes the shortest decimal in the rounding interval of x (the closest
|
||||
one if there are several, see zmij::to_shortest()). Its digits are those of a
|
||||
multiple of 10^g of the token's last digit, with g = dec_exp + 1 - q for x's
|
||||
decimal exponent dec_exp: the multiple of 10^g nearest to x if it lies in the
|
||||
interval (at most one does), else the multiple of 10^(g - 1) nearest to x
|
||||
(see token_digits_1() and token_digits_2()). For g <= 0, the token itself is
|
||||
in the interval.
|
||||
|
||||
All of it is exact in a 64-bit fixed-point number with 56 bits after the point
|
||||
(of the token's last digit): x / 10^q = 2c * 5^-q / 2^t for x = c * 2^e2 and t
|
||||
= q + 1 - e2, with at most 56 bits after the point here, and |x / 10^q - w| is
|
||||
at most half the spacing of the doubles, below 12 units. So only the low 64
|
||||
bits of 2c * 5^-q are needed, one multiplication beyond Eisel-Lemire's. An
|
||||
exact tie between two candidates and a power of two (whose interval is not
|
||||
symmetric) are left to the full conversion.
|
||||
The position is exact in a 64-bit fixed-point number with 56 bits after the
|
||||
point (of the token's last digit): x / 10^q = 2c * 5^-q / 2^t for x = c * 2^e2
|
||||
and t = q + 1 - e2, with at most 56 bits after the point here, and |x / 10^q -
|
||||
w| is at most half the spacing of the doubles, below 12 units. So only the low
|
||||
64 bits of 2c * 5^-q are needed. c and e2 come from Eisel-Lemire's product
|
||||
directly (as eisel_lemire() rounds it: no tie is possible for q < -4, and its
|
||||
second product is added where it can carry into the result).
|
||||
|
||||
@param[out] offset x / 10^q - w + 32, with 56 bits after the point
|
||||
@param[out] radius half the spacing of the doubles at x, likewise (+ 1 if
|
||||
the interval is closed)
|
||||
@return g; 3 if the digits are not decided here (a power of two, whose
|
||||
interval is not symmetric, or out of range)
|
||||
*/
|
||||
NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t shortest_token_digits(std::uint64_t w, int q, std::uint64_t bits) noexcept
|
||||
NLOHMANN_VIEW_ALWAYS_INLINE int locate_token(std::uint64_t w, int q, std::uint64_t& offset, std::uint64_t& radius) noexcept
|
||||
{
|
||||
const std::uint64_t fraction = bits & ((std::uint64_t{1} << 52u) - 1);
|
||||
const std::uint64_t c = fraction | (std::uint64_t{1} << 52u);
|
||||
const int e2 = static_cast<int>(bits >> 52u) - 1075;
|
||||
const int lz = count_leading_zeros(w);
|
||||
const std::uint64_t normalized = w << static_cast<unsigned>(lz);
|
||||
const auto index = static_cast<std::size_t>(2 * (q - pow5_128_smallest_power));
|
||||
const uint128_parts product = full_multiplication(normalized, pow5_128()[index]);
|
||||
std::uint64_t high = product.high;
|
||||
if (NLOHMANN_VIEW_UNLIKELY((high & 0x1FFu) == 0x1FFu))
|
||||
{
|
||||
// (as eisel_lemire(): the lower bits may carry into the result)
|
||||
const std::uint64_t second = full_multiplication(normalized, pow5_128()[index + 1]).high;
|
||||
high += product.low + second < product.low ? 1u : 0u;
|
||||
}
|
||||
const auto upperbit = static_cast<unsigned>(high >> 63u);
|
||||
const std::uint64_t mantissa = high >> (upperbit + 9);
|
||||
const std::uint64_t c = (mantissa + (mantissa & 1u)) >> 1u;
|
||||
// e2 as eisel_lemire() computes the exponent (log2(10) ~ 217706 / 2^16)
|
||||
const int e2 = static_cast<int>((217706 * static_cast<std::int64_t>(q)) >> 16) + 63 + static_cast<int>(upperbit) - lz - 52;
|
||||
const int t = q + 1 - e2;
|
||||
const int g = ((e2 * 315653) >> 20) + 1 - q; // (dec_exp as in zmij::to_shortest(), for a regular x)
|
||||
if (NLOHMANN_VIEW_UNLIKELY(fraction == 0 || t < 1 || t > 56 || g > 2))
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
if (g <= 0)
|
||||
{
|
||||
return w;
|
||||
}
|
||||
const unsigned s = static_cast<unsigned>(56 - t);
|
||||
const unsigned minus_q = static_cast<unsigned>(-q);
|
||||
const std::uint64_t five = ::nlohmann::detail::dtoa_impl::powers_of_ten()[minus_q] >> minus_q; // 5^-q
|
||||
constexpr std::uint64_t one = std::uint64_t{1} << 56u;
|
||||
// positions are kept 32 units above their value, so that they are not
|
||||
// negative; x and the candidates are at most 112 units apart
|
||||
constexpr std::uint64_t bias = 32;
|
||||
const std::uint64_t x = (2 * c * five) << s; // x / 10^q modulo 256 units
|
||||
const std::uint64_t half = five << s; // half the spacing of the doubles (the interval's radius)
|
||||
const bool even = (c & 1u) == 0; // (the interval is closed)
|
||||
// (the choices are masks, not conditions, which compilers could make
|
||||
// branches that these data mispredict)
|
||||
const std::uint64_t g1 = 0 - static_cast<std::uint64_t>(g == 1);
|
||||
const std::uint64_t r10 = w - ((w / 10) * 10);
|
||||
const std::uint64_t r100 = w - ((w / 100) * 100);
|
||||
// the multiple of G = 10^g nearest to x: a0 or a neighbor
|
||||
const std::uint64_t grid = (10u & g1) | (100u & ~g1);
|
||||
const std::uint64_t r = (r10 & g1) | (r100 & ~g1);
|
||||
const std::uint64_t at = x - ((w - r - bias) << 56u); // x - a0, biased
|
||||
const std::uint64_t up = 0 - static_cast<std::uint64_t>(at > (bias * one) + (grid * (one / 2)));
|
||||
const std::uint64_t down = 0 - static_cast<std::uint64_t>(at + (grid * (one / 2)) < bias * one);
|
||||
const std::uint64_t step = (grid & up) - (grid & down);
|
||||
const std::uint64_t center = (bias + step) * one;
|
||||
const std::uint64_t below = 0 - static_cast<std::uint64_t>(at < center);
|
||||
const std::uint64_t dist = ((center - at) & below) | ((at - center) & ~below);
|
||||
const std::uint64_t shorter = 0 - static_cast<std::uint64_t>(dist < half + (even ? 1u : 0u));
|
||||
const std::uint64_t a = w - r + step;
|
||||
// else the multiple of 1 (g == 1) or 10 (g == 2) nearest to x
|
||||
const std::uint64_t rl = r10 & ~g1;
|
||||
const std::uint64_t pos = x - ((w - rl - bias) << 56u) + ((one / 2) & g1) + ((5 * one) & ~g1); // x - b0 + half a step, biased
|
||||
const std::uint64_t whole = pos >> 56u; // in [20, 59]
|
||||
const std::uint64_t m = whole + 20 - bias; // (a multiple of 10 above whole - bias)
|
||||
const std::uint64_t tens = (m * 205) >> 11u; // m / 10
|
||||
const std::uint64_t tie = 0 - static_cast<std::uint64_t>((pos & (one - 1)) == 0 && (g == 1 || m == tens * 10));
|
||||
const std::uint64_t b = ((w + whole - bias) & g1) | ((w - rl + (tens * 10) - 20) & ~g1);
|
||||
return (a & shorter) | (b & ~tie & ~shorter);
|
||||
static const std::array<std::uint64_t, 17> fives = {{1u, 5u, 25u, 125u, 625u, 3125u, 15625u, 78125u, 390625u, 1953125u, 9765625u, 48828125u, 244140625u, 1220703125u, 6103515625u, 30517578125u, 152587890625u}};
|
||||
const std::uint64_t five = fives[static_cast<std::size_t>(-q)]; // 5^-q
|
||||
const unsigned s = static_cast<unsigned>(56 - t) & 63u;
|
||||
offset = ((2 * c * five) << s) - (w << 56u) + (std::uint64_t{32} << 56u);
|
||||
radius = (five << s) + ((c & 1u) == 0 ? 1u : 0u);
|
||||
// (c = 2^52: a power of two; c = 2^53: rounded up to one. With 8 to 16
|
||||
// fraction digits, as the batch's tokens have, t is in [15, 40] and g at
|
||||
// most 2. x lies halfway between two candidates only if its distance from
|
||||
// the token is a whole unit (g == 2) or a half (g == 1): such tokens are
|
||||
// left to the full conversion)
|
||||
const bool decided = c - ((std::uint64_t{1} << 52u) + 1) < (std::uint64_t{1} << 52u) - 1
|
||||
&& (offset & ((std::uint64_t{1} << 55u) - 1)) != 0;
|
||||
return decided ? g : 3;
|
||||
}
|
||||
|
||||
/// the eight ASCII digits of x < 10^8, the first in the lowest byte (as
|
||||
/// little-endian memory holds them)
|
||||
NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t eight_digit_text(std::uint64_t x) noexcept
|
||||
/// the distance of a position (see locate_token()) from the biased point 32
|
||||
NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t biased_distance(std::uint64_t position) noexcept
|
||||
{
|
||||
const std::uint64_t abcd = (x * 109951163u) >> 40u; // x / 10000
|
||||
std::uint64_t v = abcd | ((x - (abcd * 10000u)) << 32u);
|
||||
const std::uint64_t ab = ((v * 10486u) >> 20u) & 0x0000007F0000007Fu; // / 100 per 32-bit lane
|
||||
v = ab | ((v - (ab * 100u)) << 16u);
|
||||
const std::uint64_t a = ((v * 103u) >> 10u) & 0x000F000F000F000Fu; // / 10 per 16-bit lane
|
||||
constexpr std::uint64_t center = std::uint64_t{32} << 56u;
|
||||
return position >= center ? position - center : center - position;
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief (g == 1) the digits dump() writes for a token, from the position of its
|
||||
double (see locate_token()): the multiple of 10 nearest to x if it is in the
|
||||
interval, else w rounded to x (an exact tie between two candidates, like
|
||||
754570942.33984373, whose double is 754570942.33984375, is excluded by
|
||||
locate_token())
|
||||
|
||||
With |x - w| at most the radius, which is below half the spacing of the
|
||||
candidates, the multiple of 10 nearest to x is w - r10 or the one above it.
|
||||
*/
|
||||
NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t token_digits_1(std::uint64_t w, std::uint64_t offset, std::uint64_t radius, std::uint64_t r10) noexcept
|
||||
{
|
||||
constexpr std::uint64_t one = std::uint64_t{1} << 56u;
|
||||
const std::uint64_t from = offset + (r10 * one); // x - (w - r10) (+ 32)
|
||||
const bool up = from > 37 * one; // (nearer to the multiple above)
|
||||
const bool shorter = biased_distance(from - (up ? 10 * one : 0)) < radius;
|
||||
const std::uint64_t a = w - r10 + (up ? 10u : 0u);
|
||||
const std::uint64_t b = w + ((offset + (one / 2)) >> 56u) - 32; // w rounded to x
|
||||
return shorter ? a : b;
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief (g == 2) the digits dump() writes for a token, as token_digits_1(): the
|
||||
multiple of 100 nearest to x if it is in the interval, else the multiple of 10
|
||||
nearest to x
|
||||
*/
|
||||
NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t token_digits_2(std::uint64_t w, std::uint64_t offset, std::uint64_t radius, std::uint64_t r10, std::uint64_t r100) noexcept
|
||||
{
|
||||
constexpr std::uint64_t one = std::uint64_t{1} << 56u;
|
||||
const std::uint64_t from = offset + (r100 * one);
|
||||
const bool up = from > 82 * one;
|
||||
const bool shorter = biased_distance(from - (up ? 100 * one : 0)) < radius;
|
||||
const std::uint64_t a = w - r100 + (up ? 100u : 0u);
|
||||
// x - (w - r10), in [-12, 21]: the multiple of 10 nearest to x is that one,
|
||||
// or one or two above, or one below
|
||||
const std::uint64_t y = offset + (r10 * one);
|
||||
const std::uint64_t steps = (y > 37 * one ? 1u : 0u) + (y > 47 * one ? 1u : 0u);
|
||||
const bool down = y < 27 * one;
|
||||
const std::uint64_t b = w - r10 + (10 * steps) - (down ? 10u : 0u);
|
||||
return shorter ? a : b;
|
||||
}
|
||||
|
||||
/// the four ASCII digits of x < 10^4, the first in the lowest byte (as
|
||||
/// little-endian memory holds them)
|
||||
NLOHMANN_VIEW_ALWAYS_INLINE std::uint32_t four_digit_text(std::uint32_t x) noexcept
|
||||
{
|
||||
const std::uint32_t ab = (x * 5243u) >> 19u; // x / 100
|
||||
std::uint32_t v = ab | ((x - (ab * 100u)) << 16u);
|
||||
const std::uint32_t a = ((v * 103u) >> 10u) & 0x000F000Fu; // / 10 per 16-bit lane
|
||||
v = a | ((v - (a * 10u)) << 8u);
|
||||
return v + 0x3030303030303030u;
|
||||
return v + 0x30303030u;
|
||||
}
|
||||
|
||||
/// how the view's dump() writes a value
|
||||
@@ -7361,7 +7478,7 @@ class view_serializer
|
||||
else if (enabled(fast_double::value))
|
||||
{
|
||||
room(64);
|
||||
w = write_double_ahead(w, n, batch);
|
||||
w = write_double_ahead(w, src, n, batch);
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -7386,6 +7503,20 @@ class view_serializer
|
||||
case value_t::array:
|
||||
{
|
||||
const bool object = n->kind == static_cast<std::uint8_t>(value_t::object);
|
||||
// (read-only documents, compact output) a pair of floats,
|
||||
// like the coordinates of GeoJSON, without a frame
|
||||
if (!Editable && !Pretty && !SourceNumbers && enabled(fast_double::value) && n->next == 3 && n->len == 2
|
||||
&& n[1].kind == static_cast<std::uint8_t>(value_t::number_float) && n[2].kind == static_cast<std::uint8_t>(value_t::number_float))
|
||||
{
|
||||
room(2 * 64);
|
||||
*w++ = '[';
|
||||
w = write_double_ahead(w, src, n + 1, batch);
|
||||
*w++ = ',';
|
||||
w = write_double_ahead(w, src, n + 2, batch);
|
||||
*w++ = ']';
|
||||
n += 2; // (the element before the next node)
|
||||
break;
|
||||
}
|
||||
room(8);
|
||||
if (n->len == 0)
|
||||
{
|
||||
@@ -7495,13 +7626,31 @@ class view_serializer
|
||||
}
|
||||
}
|
||||
|
||||
/// (float_batch) the mark of digits whose rounding carried beyond the
|
||||
/// token's last four (see mark())
|
||||
static constexpr std::uint64_t carried() noexcept
|
||||
{
|
||||
return std::uint64_t{1} << 63u;
|
||||
}
|
||||
|
||||
/// floats of 16 or 17 digits ahead of the walk, converted together (see
|
||||
/// convert_ahead()); the digits are those of shortest_token_digits(), 0
|
||||
/// for a float the batch does not write
|
||||
/// convert_ahead()), with their digits as mark() keeps them
|
||||
struct float_batch
|
||||
{
|
||||
std::array<const node*, 32> at; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read
|
||||
std::array<std::uint64_t, 32> digits; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read
|
||||
static constexpr std::size_t capacity() noexcept
|
||||
{
|
||||
return 256;
|
||||
}
|
||||
/// (on the heap, allocated by the first conversion: a frame of more
|
||||
/// than a page made every dump of a small document slower)
|
||||
struct entries
|
||||
{
|
||||
std::array<const node*, 256> at; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read
|
||||
std::array<std::uint64_t, 256> digits; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read
|
||||
};
|
||||
std::unique_ptr<entries> storage{};
|
||||
const node** at = nullptr;
|
||||
std::uint64_t* digits = nullptr;
|
||||
std::size_t count = 0;
|
||||
std::size_t next = 0;
|
||||
const node* scanned = nullptr; ///< where the last scan of the node array ended
|
||||
@@ -7509,21 +7658,24 @@ class view_serializer
|
||||
|
||||
/// (doubles) the float at n as dump() writes it, at w (64 bytes of room):
|
||||
/// from the batch if it holds n
|
||||
NLOHMANN_VIEW_ALWAYS_INLINE char* write_double_ahead(char* w, const node* n, float_batch& batch)
|
||||
NLOHMANN_VIEW_ALWAYS_INLINE char* write_double_ahead(char* w, const char* src, const node* n, float_batch& batch)
|
||||
{
|
||||
#if NLOHMANN_VIEW_NEON
|
||||
if (batch.next != batch.count && batch.at[batch.next] == n && batch.digits[batch.next] != 0)
|
||||
#if NLOHMANN_VIEW_VECTOR && NLOHMANN_VIEW_LITTLE_ENDIAN
|
||||
// (digits 0: not written by the batch; carried(): rounding carried
|
||||
// beyond the last four digits)
|
||||
if (batch.next != batch.count && batch.at[batch.next] == n && batch.digits[batch.next] - 1 < carried() - 1)
|
||||
{
|
||||
return write_token_digits(w, m_doc.src + n->off, n->len, n->extra >> 8u, batch.digits[batch.next++]);
|
||||
return write_token_digits(w, src, *n, batch.digits[batch.next++]);
|
||||
}
|
||||
return write_double_beside(w, n, batch);
|
||||
#else
|
||||
static_cast<void>(src);
|
||||
static_cast<void>(batch);
|
||||
return write_double_at(w, *n);
|
||||
#endif
|
||||
}
|
||||
|
||||
#if NLOHMANN_VIEW_NEON
|
||||
#if NLOHMANN_VIEW_VECTOR && NLOHMANN_VIEW_LITTLE_ENDIAN
|
||||
/// write_double_ahead() for a float that is not the batch's next one (out
|
||||
/// of line: the walk is faster with less code in its loop)
|
||||
NLOHMANN_VIEW_NOINLINE char* write_double_beside(char* w, const node* n, float_batch& batch)
|
||||
@@ -7537,9 +7689,13 @@ class view_serializer
|
||||
if ((batch.next != batch.count && batch.at[batch.next] == n) || convert_ahead(n, batch))
|
||||
{
|
||||
const std::uint64_t digits = batch.digits[batch.next++];
|
||||
if ((digits & carried()) != 0)
|
||||
{
|
||||
return write_carried_digits(w, *n, digits & ~carried());
|
||||
}
|
||||
if (digits != 0)
|
||||
{
|
||||
return write_token_digits(w, m_doc.src + n->off, n->len, n->extra >> 8u, digits);
|
||||
return write_token_digits(w, m_doc.src, *n, digits);
|
||||
}
|
||||
}
|
||||
return write_double_at(w, *n);
|
||||
@@ -7550,16 +7706,14 @@ class view_serializer
|
||||
array at once; whether n is the first of them
|
||||
|
||||
Such tokens (like the coordinates of GeoJSON, written with 17 digits) are
|
||||
written as the token with its last digits replaced (write_token_digits()).
|
||||
Each step (the digits, Eisel-Lemire, shortest_token_digits()) is a long
|
||||
chain of dependent multiplications; one loop per step over many floats lets
|
||||
the processor overlap the floats, which it does not across the steps of
|
||||
one float (on Apple M1, the dump of canada.json takes 30 % less time than
|
||||
with a conversion of each float, of which 10 points are this). Each node of the array
|
||||
written as the token with its last four digits replaced
|
||||
(write_token_digits()). Each step (the digits, locate_token(),
|
||||
token_digits_1/2()) is a long chain of dependent operations; one loop per
|
||||
step over many floats lets the processor overlap the floats, which it does
|
||||
not across the steps of one float (on Apple M1, one loop for the digits and
|
||||
locate_token() made the dump of canada.json 12 % slower). Each node of the array
|
||||
is scanned once; a node that the walk does not reach in order (in an
|
||||
editable document) is written without the batch. (AArch64 only: on x86-64
|
||||
(Haswell, GCC 13 and Clang 18), the batch was not faster than the
|
||||
conversion of each float.)
|
||||
editable document) is written without the batch.
|
||||
*/
|
||||
NLOHMANN_VIEW_NOINLINE bool convert_ahead(const node* n, float_batch& batch)
|
||||
{
|
||||
@@ -7571,72 +7725,144 @@ class view_serializer
|
||||
}
|
||||
const char* const src = m_doc.src;
|
||||
const char* const src_end = src + m_doc.size;
|
||||
const node* const stop = last - n > 128 ? n + 128 : last;
|
||||
const node* const stop = last - n > 1024 ? n + 1024 : last;
|
||||
batch.count = 0;
|
||||
batch.next = 0;
|
||||
// the digits of each token (and its last four from the text)
|
||||
std::array<std::uint64_t, 256> digits; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read
|
||||
std::array<int, 256> exponent; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read
|
||||
std::array<std::uint32_t, 256> last_four; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read
|
||||
const node* p = n;
|
||||
for (; p != stop && batch.count != batch.at.size(); ++p)
|
||||
if (!batch.storage)
|
||||
{
|
||||
batch.storage.reset(new typename float_batch::entries); // NOLINT(cppcoreguidelines-owning-memory,modernize-make-unique): C++11
|
||||
batch.at = batch.storage->at.data();
|
||||
batch.digits = batch.storage->digits.data();
|
||||
}
|
||||
for (; p != stop && batch.count != float_batch::capacity(); ++p)
|
||||
{
|
||||
// a token of the source: 16 or 17 digits, at least 8 after the
|
||||
// point, no exponent, no leading zero, 32 bytes to read
|
||||
// point, no exponent, 32 bytes to read
|
||||
const unsigned frac_digits = p->extra >> 8u;
|
||||
const unsigned count = (p->extra & 0xFFu) + frac_digits;
|
||||
const unsigned int_digits = p->extra & 0xFFu;
|
||||
const unsigned count = int_digits + frac_digits;
|
||||
if (p->kind == static_cast<std::uint8_t>(value_t::number_float) && (p->flags & node_flags::storage) == 0
|
||||
&& count - 16u <= 1u && frac_digits >= 8 && src_end - (src + p->off) >= 33)
|
||||
{
|
||||
const char* const token = src + p->off;
|
||||
const auto* const token = reinterpret_cast<const unsigned char*>(src + p->off); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast)
|
||||
const unsigned sign = token[0] == '-' ? 1 : 0;
|
||||
if (p->len == sign + count + 1 && token[sign] != '0')
|
||||
{
|
||||
batch.at[batch.count++] = p;
|
||||
}
|
||||
const unsigned char* const first = token + sign;
|
||||
const std::uint64_t w = sixteen_digits(first, int_digits);
|
||||
// (the 17th digit follows the point; a token with an exponent
|
||||
// gets digits outside the range of locate_token(), and is not
|
||||
// written by the batch. A token 0.xxx with 16 significant
|
||||
// digits is: its double is written as "0." and its digits too)
|
||||
const std::uint64_t w17 = (w * 10) + static_cast<std::uint64_t>(first[17] - '0');
|
||||
const std::size_t i = batch.count++;
|
||||
batch.at[i] = p;
|
||||
digits[i] = p->len != sign + count + 1 ? 0 : (count == 17 ? w17 : w);
|
||||
exponent[i] = -static_cast<int>(frac_digits);
|
||||
// (byte 0: the last digit, byte 1: the last two, bits 16-29: the last four)
|
||||
std::uint32_t four = 0;
|
||||
std::memcpy(&four, token + p->len - 4, 4);
|
||||
four -= 0x30303030u;
|
||||
const std::uint32_t pairs = ((four & 0x0F0F0F0Fu) * 2561u) >> 8u; // (byte 0, 2: the pairs)
|
||||
last_four[i] = (four >> 24u) | ((pairs >> 16u & 0xFFu) << 8u) | ((((pairs & 0x00FF00FFu) * 6553601u) >> 16u) << 16u);
|
||||
}
|
||||
}
|
||||
batch.scanned = p;
|
||||
// the position of each double, and the floats of either kind
|
||||
std::array<std::uint64_t, 256> offset; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read
|
||||
std::array<std::uint64_t, 256> radius; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read
|
||||
std::array<std::uint8_t, 256> kind_1; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read
|
||||
std::array<std::uint8_t, 256> kind_2; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read
|
||||
std::size_t count_1 = 0;
|
||||
std::size_t count_2 = 0;
|
||||
for (std::size_t i = 0; i != batch.count; ++i)
|
||||
{
|
||||
const node& f = *batch.at[i];
|
||||
const auto* digits = reinterpret_cast<const unsigned char*>(src + f.off); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast)
|
||||
digits += *digits == '-' ? 1 : 0;
|
||||
const std::uint64_t w = sixteen_digits(digits, f.extra & 0xFFu);
|
||||
// (the 17th digit follows the point)
|
||||
batch.digits[i] = (f.extra & 0xFFu) + (f.extra >> 8u) == 17 ? (w * 10) + static_cast<std::uint64_t>(digits[17] - '0') : w;
|
||||
const std::uint64_t w = digits[i];
|
||||
// (the range excludes the values of bytes that are not digits, only
|
||||
// in a damaged image loaded with image_check::bounds)
|
||||
const int g = w - 1000000000000000u < 99000000000000000u ? locate_token(w, exponent[i], offset[i], radius[i]) : 3;
|
||||
batch.digits[i] = g <= 0 ? (last_four[i] >> 16u) + 1 : 0;
|
||||
kind_1[count_1] = static_cast<std::uint8_t>(i);
|
||||
count_1 += g == 1 ? 1 : 0;
|
||||
kind_2[count_2] = static_cast<std::uint8_t>(i);
|
||||
count_2 += g == 2 ? 1 : 0;
|
||||
}
|
||||
std::array<std::uint64_t, 32> bits; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read
|
||||
for (std::size_t i = 0; i != batch.count; ++i)
|
||||
// the digits to write, in one loop per kind (see mark())
|
||||
for (std::size_t k = 0; k != count_1; ++k)
|
||||
{
|
||||
bits[i] = eisel_lemire<ieee_binary_format<53>>(-static_cast<int>(batch.at[i]->extra >> 8u), batch.digits[i]);
|
||||
const std::size_t i = kind_1[k];
|
||||
const std::uint64_t s = token_digits_1(digits[i], offset[i], radius[i], last_four[i] & 0xFFu);
|
||||
batch.digits[i] = mark(digits[i], s, last_four[i] >> 16u);
|
||||
}
|
||||
for (std::size_t i = 0; i != batch.count; ++i)
|
||||
for (std::size_t k = 0; k != count_2; ++k)
|
||||
{
|
||||
const std::uint64_t w = batch.digits[i];
|
||||
// (the range also excludes the values of bytes that are not
|
||||
// digits, only in a damaged image loaded with image_check::bounds)
|
||||
const std::uint64_t s = w - 1000000000000000u < 99000000000000000u ? shortest_token_digits(w, -static_cast<int>(batch.at[i]->extra >> 8u), bits[i]) : 0;
|
||||
// (only the last eight digits are written over the token's)
|
||||
batch.digits[i] = s / 100000000u == w / 100000000u ? s : 0;
|
||||
const std::size_t i = kind_2[k];
|
||||
const std::uint64_t s = token_digits_2(digits[i], offset[i], radius[i], last_four[i] & 0xFFu, (last_four[i] >> 8u) & 0xFFu);
|
||||
batch.digits[i] = mark(digits[i], s, last_four[i] >> 16u);
|
||||
}
|
||||
return batch.count != 0 && batch.at[0] == n;
|
||||
}
|
||||
|
||||
/// a float token of 16 or 17 digits as dump() writes it, at w (64 bytes
|
||||
/// of room): the token with its last eight digits replaced by those of
|
||||
/// digits (see shortest_token_digits(); the others are the token's), and
|
||||
/// without trailing zeros after the point, but one
|
||||
static char* write_token_digits(char* w, const char* token, std::uint32_t len, unsigned frac_digits, std::uint64_t digits) noexcept
|
||||
/*!
|
||||
@brief a float token of 16 or 17 digits as dump() writes it, at w (64 bytes
|
||||
of room), from the digits of its double (see locate_token()) whose
|
||||
rounding carried beyond the token's last four digits
|
||||
|
||||
(like 43.42027299999999 -> 43.420273): the digits go to the writer of
|
||||
Zmij's results as its 15 or 16 leading digits and one more, which omits
|
||||
the trailing zeros.
|
||||
*/
|
||||
NLOHMANN_VIEW_NOINLINE char* write_carried_digits(char* w, const node& n, std::uint64_t digits) const noexcept
|
||||
{
|
||||
const int exponent = -static_cast<int>(n.extra >> 8u);
|
||||
const std::uint64_t leading = digits / 10;
|
||||
const auto last = static_cast<unsigned char>(digits - (leading * 10));
|
||||
const bool seventeen = digits >= 10000000000000000u;
|
||||
*w = '-';
|
||||
w += m_doc.src[n.off] == '-' ? 1 : 0;
|
||||
return ::nlohmann::detail::dtoa_impl::write_shortest(w, seventeen ? ::nlohmann::detail::zmij::shortest_decimal{leading, exponent, last, last != 0}
|
||||
: ::nlohmann::detail::zmij::shortest_decimal{digits, exponent - 1, 0, false});
|
||||
}
|
||||
|
||||
/// the digits s of a token with the digits w and the last four digits
|
||||
/// r4, as the batch keeps them: the new last four digits + 1 (only those
|
||||
/// are written over the token's), or s marked if it differs from w beyond
|
||||
/// them (0 stays for a float the batch does not write)
|
||||
static std::uint64_t mark(std::uint64_t w, std::uint64_t s, std::uint64_t r4) noexcept
|
||||
{
|
||||
const std::uint64_t last = s - (w - r4);
|
||||
return last < 10000 ? last + 1 : s | carried();
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief a float token of 16 or 17 digits as dump() writes it, at w (64 bytes
|
||||
of room): the token with its last four digits replaced by digits - 1 (see
|
||||
mark(); the others are the token's), and without trailing zeros after the
|
||||
point, but one
|
||||
*/
|
||||
char* write_token_digits(char* w, const char* src, const node& n, std::uint64_t digits)
|
||||
{
|
||||
const char* const token = src + n.off;
|
||||
std::memcpy(w, token, 32);
|
||||
char* const end = w + len;
|
||||
const std::uint64_t last = eight_digit_text(digits % 100000000u);
|
||||
std::memcpy(end - 8, &last, 8);
|
||||
char* const end = w + n.len;
|
||||
const std::uint32_t last = four_digit_text(static_cast<std::uint32_t>(digits - 1));
|
||||
std::memcpy(end - 4, &last, 4);
|
||||
// (the token's text before the new digits, from the source: reading
|
||||
// the output back would wait for the stores)
|
||||
std::uint64_t before = 0;
|
||||
std::memcpy(&before, token + len - 16, 8);
|
||||
const std::uint64_t last_zeros = last ^ 0x3030303030303030u;
|
||||
std::memcpy(&before, token + n.len - 12, 8);
|
||||
const std::uint32_t last_zeros = last ^ 0x30303030u;
|
||||
const std::uint64_t before_zeros = before ^ 0x3030303030303030u;
|
||||
const unsigned zeros = last_zeros != 0 ? static_cast<unsigned>(count_leading_zeros(last_zeros)) / 8
|
||||
: 8 + (before_zeros != 0 ? static_cast<unsigned>(count_leading_zeros(before_zeros)) / 8 : 8);
|
||||
if (NLOHMANN_VIEW_UNLIKELY(last_zeros == 0 && before_zeros == 0))
|
||||
{
|
||||
return write_double_at(w, n); // (twelve or more trailing zeros)
|
||||
}
|
||||
const unsigned zeros = last_zeros != 0 ? static_cast<unsigned>(count_leading_zeros(last_zeros)) / 8 - 4
|
||||
: 4 + (static_cast<unsigned>(count_leading_zeros(before_zeros)) / 8);
|
||||
const unsigned frac_digits = n.extra >> 8u;
|
||||
return end - (zeros < frac_digits ? zeros : frac_digits - 1);
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -151,10 +151,12 @@ TEST_CASE("json_view dump")
|
||||
}
|
||||
|
||||
// tokens of 16 and 17 digits without an exponent (like GeoJSON
|
||||
// coordinates), which dump() converts in batches: random digits, and
|
||||
// digits near a shorter decimal (...99999997, ...00000003), whose
|
||||
// rounding carries; in pairs and among other values, so that the
|
||||
// batches are interrupted; the last token near the end of the coordinates
|
||||
// coordinates), which dump() converts in batches: doubles exactly
|
||||
// halfway between two candidates for the shortest digits, random
|
||||
// digits, and digits near a shorter decimal (...99999997,
|
||||
// ...00000003), whose rounding carries; in pairs and among other
|
||||
// values, so that the batches are interrupted; the last token near the
|
||||
// end of the text
|
||||
{
|
||||
std::mt19937_64 tokens(5627); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed)
|
||||
const auto draw = [&tokens](std::size_t n)
|
||||
@@ -162,7 +164,8 @@ TEST_CASE("json_view dump")
|
||||
const std::uint64_t r = tokens() % n;
|
||||
return static_cast<std::size_t>(r);
|
||||
};
|
||||
std::string coordinates = "[";
|
||||
std::string coordinates = "[754570942.33984373,257769445.56835938,858106901.94921875,97944964.876953125,"
|
||||
"[0.5,1e-7],[1.5,-0.0,2.5],[1E2,12.5,0.1,3.25,4.5,5.75,6.125,7.0,8.5],[],[1.5],[1,2.5],[2.5,\"x\"],[[1.5],2.5],";
|
||||
for (int i = 0; i < 20000; ++i)
|
||||
{
|
||||
const std::size_t length = 16 + draw(2);
|
||||
@@ -191,7 +194,7 @@ TEST_CASE("json_view dump")
|
||||
{
|
||||
const ordered_json& e = j[i++];
|
||||
CHECK(v.dump() == e.dump());
|
||||
if (v.is_array())
|
||||
if (v.is_array() && v.size() != 0 && v[0].is_number())
|
||||
{
|
||||
CHECK(v[0].get<double>() == e[0].get<double>());
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user