diff --git a/include/nlohmann/detail/view/builder.hpp b/include/nlohmann/detail/view/builder.hpp index 89b5af2ab..31c78b996 100644 --- a/include/nlohmann/detail/view/builder.hpp +++ b/include/nlohmann/detail/view/builder.hpp @@ -1082,7 +1082,7 @@ indent_done: { ++p; const unsigned char* const f0 = p; - p = skip_digits(p, e); + p = skip_fraction_digits(p, e); if (NLOHMANN_VIEW_UNLIKELY(p == f0)) { return fail(error_code::number_after_dot); diff --git a/include/nlohmann/detail/view/scan.hpp b/include/nlohmann/detail/view/scan.hpp index b8d862203..9cea6b05c 100644 --- a/include/nlohmann/detail/view/scan.hpp +++ b/include/nlohmann/detail/view/scan.hpp @@ -198,6 +198,57 @@ inline const unsigned char* find_block_comment_end(const unsigned char* p, const return p; } +/// The digits of a fraction, whose number varies from one number to the next: skip_digits() with 16 bytes compared at +/// once and the end of the run taken from a bit mask (SSE2), instead of a compare and a branch for each digit. That +/// is about a third of the instructions for a fraction of 12 digits, and no branch that is mispredicted when the +/// lengths vary. Where there is no SSE2, skip_digits() is used. +#if NLOHMANN_VIEW_SSE2 +// The last bytes of the input, one at a time. GCC unrolls this loop in every inlined copy of the function below, which +// makes the parse loop 4 KiB larger and slows down documents that have hardly any fractions; out of line, it does +// not. Clang does better with the loop inline (a call in the parse loop costs it more than the unrolling). +#if defined(__GNUC__) && !defined(__clang__) +inline NLOHMANN_VIEW_NOINLINE const unsigned char* skip_digits_tail(const unsigned char* p, const unsigned char* e) noexcept +{ + while (p != e && is_digit(*p)) + { + ++p; + } + return p; +} +#else +NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* skip_digits_tail(const unsigned char* p, const unsigned char* e) noexcept +{ + while (p != e && is_digit(*p)) + { + ++p; + } + return p; +} +#endif + +NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* skip_fraction_digits(const unsigned char* p, const unsigned char* e) noexcept +{ + while (e - p >= 16) + { + const __m128i in = _mm_loadu_si128(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) + // a digit is above '/' and below ':' (signed compares: a byte with the top bit set is negative, and neither) + const __m128i digits = _mm_and_si128(_mm_cmpgt_epi8(in, _mm_set1_epi8('/')), _mm_cmplt_epi8(in, _mm_set1_epi8(':'))); + const auto not_digit = static_cast(~static_cast(_mm_movemask_epi8(digits)) & 0xFFFFu); + if (not_digit != 0) + { + return p + count_trailing_zeros(not_digit); + } + p += 16; + } + return skip_digits_tail(p, e); +} +#else +NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* skip_fraction_digits(const unsigned char* p, const unsigned char* e) noexcept +{ + return skip_digits(p, e); +} +#endif + /// powers of ten up to 10^19 as integers inline std::uint64_t int_pow10(unsigned k) noexcept { diff --git a/single_include/nlohmann/json_view.hpp b/single_include/nlohmann/json_view.hpp index 8ff21c77f..740fef708 100644 --- a/single_include/nlohmann/json_view.hpp +++ b/single_include/nlohmann/json_view.hpp @@ -1218,6 +1218,57 @@ inline const unsigned char* find_block_comment_end(const unsigned char* p, const return p; } +/// The digits of a fraction, whose number varies from one number to the next: skip_digits() with 16 bytes compared at +/// once and the end of the run taken from a bit mask (SSE2), instead of a compare and a branch for each digit. That +/// is about a third of the instructions for a fraction of 12 digits, and no branch that is mispredicted when the +/// lengths vary. Where there is no SSE2, skip_digits() is used. +#if NLOHMANN_VIEW_SSE2 +// The last bytes of the input, one at a time. GCC unrolls this loop in every inlined copy of the function below, which +// makes the parse loop 4 KiB larger and slows down documents that have hardly any fractions; out of line, it does +// not. Clang does better with the loop inline (a call in the parse loop costs it more than the unrolling). +#if defined(__GNUC__) && !defined(__clang__) +inline NLOHMANN_VIEW_NOINLINE const unsigned char* skip_digits_tail(const unsigned char* p, const unsigned char* e) noexcept +{ + while (p != e && is_digit(*p)) + { + ++p; + } + return p; +} +#else +NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* skip_digits_tail(const unsigned char* p, const unsigned char* e) noexcept +{ + while (p != e && is_digit(*p)) + { + ++p; + } + return p; +} +#endif + +NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* skip_fraction_digits(const unsigned char* p, const unsigned char* e) noexcept +{ + while (e - p >= 16) + { + const __m128i in = _mm_loadu_si128(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) + // a digit is above '/' and below ':' (signed compares: a byte with the top bit set is negative, and neither) + const __m128i digits = _mm_and_si128(_mm_cmpgt_epi8(in, _mm_set1_epi8('/')), _mm_cmplt_epi8(in, _mm_set1_epi8(':'))); + const auto not_digit = static_cast(~static_cast(_mm_movemask_epi8(digits)) & 0xFFFFu); + if (not_digit != 0) + { + return p + count_trailing_zeros(not_digit); + } + p += 16; + } + return skip_digits_tail(p, e); +} +#else +NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* skip_fraction_digits(const unsigned char* p, const unsigned char* e) noexcept +{ + return skip_digits(p, e); +} +#endif + /// powers of ten up to 10^19 as integers inline std::uint64_t int_pow10(unsigned k) noexcept { @@ -2324,7 +2375,7 @@ indent_done: { ++p; const unsigned char* const f0 = p; - p = skip_digits(p, e); + p = skip_fraction_digits(p, e); if (NLOHMANN_VIEW_UNLIKELY(p == f0)) { return fail(error_code::number_after_dot); diff --git a/tests/src/unit-json_view_builder.cpp b/tests/src/unit-json_view_builder.cpp index 4023c7848..945e6b638 100644 --- a/tests/src/unit-json_view_builder.cpp +++ b/tests/src/unit-json_view_builder.cpp @@ -523,6 +523,47 @@ TEST_CASE("json_view builder: strings across vector blocks") } } +TEST_CASE("json_view builder: fraction digits") +{ + // The digits of a fraction are skipped 16 bytes at a time where there are + // that many left (SSE2), and one at a time after that. Fractions of every + // length from 0 to 40 digits, followed by every kind of byte that ends a + // number or lies next to the digits in ASCII ('/' and ':'), as values, as + // keys' neighbours, and at the very end of the text (a vector without a + // terminating NUL). The length of the number is part of the value that + // the test compares with json::parse(). + const std::string followers = ",]}/:.eE+-x \n\t\"09"; + std::mt19937 rng(2028); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed): reproducible + for (std::size_t digits = 0; digits <= 40; ++digits) + { + for (int i = 0; i < 3; ++i) + { + std::string fraction; + for (std::size_t k = 0; k < digits; ++k) + { + fraction += static_cast('0' + (rng() % 10)); + } + const std::string number = std::string(rng() % 2 == 0 ? "-" : "") + std::to_string(rng() % 100000) + "." + fraction; + check_same(number); + check_same("[" + number + "]"); + check_same("[" + number + "," + number + "]"); + check_same("{\"a\":" + number + "}"); + for (const char follower : followers) + { + check_same(number + follower); + check_same("[" + number + follower + "]"); + check_same("[" + number + follower + std::string(rng() % 20, '1')); + } + // everything near the end of the text: the number cut in every place + for (std::size_t cut = 1; cut <= 18 && cut <= number.size(); ++cut) + { + check_same(number.substr(0, number.size() - cut)); + check_same("[" + number.substr(0, number.size() - cut)); + } + } + } +} + TEST_CASE("json_view builder: escapes between runs of text") { // The text between escapes is copied 16 bytes at a time where there are