Select the Zmij conversion by a trait, not by the double overload

The non-template overloads for double asserted binary64 doubles wherever json.hpp was included, so the library no longer compiled where double is not IEEE 754 binary64 (AVR, -fshort-double). A trait now picks Zmij for any binary64 type, including a long double of that format (MSVC, Apple Arm), and Grisu2 for the others. Remove the unused write_short_decimal, powers_of_ten_16, zmij::decimal, zmij::to_decimal, and shortest_digits(double).

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

include/nlohmann/detail/conversions/to_chars.hpp

@@ -939,88 +939,6 @@ void grisu2(char* buf, int& len, int& decimal_exponent, FloatType value)
     grisu2(buf, len, decimal_exponent, w.minus, w.w, w.plus);
 }
 
-/*!
-@brief the shortest digits of a positive finite float (other than double): Grisu2
-*/
-template<typename FloatType>
-JSON_HEDLEY_NON_NULL(1)
-void shortest_digits(char* buf, int& len, int& decimal_exponent, FloatType value)
-{
-    grisu2(buf, len, decimal_exponent, value);
-}
-
-/*!
-@brief the shortest digits of a positive finite double: the conversion of
-Zmij (see zmij.hpp), which always finds the shortest digits that read back as
-the same value (Grisu2 does not for about one double in a thousand), and the
-closest of them if there are several
-
-v = buf * 10^decimal_exponent, as for grisu2()
-*/
-JSON_HEDLEY_NON_NULL(1)
-inline void shortest_digits(char* buf, int& len, int& decimal_exponent, double value)
-{
-    static_assert(std::numeric_limits<double>::is_iec559 && std::numeric_limits<double>::digits == 53,
-                  "internal error: the conversion of Zmij needs IEEE 754 binary64 doubles");
-    JSON_ASSERT(std::isfinite(value));
-    JSON_ASSERT(value > 0);
-
-    std::uint64_t bits = 0;
-    std::memcpy(&bits, &value, sizeof(bits));
-    zmij::decimal d = zmij::to_decimal(bits);
-    // without trailing zeros (up to 16): 8, 4, 2, 1 at a time
-    while (d.significand % 100000000 == 0)
-    {
-        d.significand /= 100000000;
-        d.exponent += 8;
-    }
-    if (d.significand % 10000 == 0)
-    {
-        d.significand /= 10000;
-        d.exponent += 4;
-    }
-    if (d.significand % 100 == 0)
-    {
-        d.significand /= 100;
-        d.exponent += 2;
-    }
-    if (d.significand % 10 == 0)
-    {
-        d.significand /= 10;
-        d.exponent += 1;
-    }
-    // at most 17 digits, written from the back two at a time
-    static constexpr const char* pairs =
-        "00010203040506070809101112131415161718192021222324252627282930313233343536373839"
-        "40414243444546474849505152535455565758596061626364656667686970717273747576777879"
-        "8081828384858687888990919293949596979899";
-    std::array<char, 20> digits{};
-    std::size_t n = digits.size();
-    while (d.significand >= 100)
-    {
-        const std::uint64_t two_digits = d.significand % 100; // a variable: GCC calls a cast of the remainder useless where std::uint64_t is std::size_t
-        const auto i = static_cast<std::size_t>(two_digits) * 2;
-        d.significand /= 100;
-        n -= 2;
-        digits[n] = pairs[i];
-        digits[n + 1] = pairs[i + 1];
-    }
-    if (d.significand >= 10)
-    {
-        const auto i = static_cast<std::size_t>(d.significand) * 2;
-        n -= 2;
-        digits[n] = pairs[i];
-        digits[n + 1] = pairs[i + 1];
-    }
-    else
-    {
-        digits[--n] = static_cast<char>('0' + d.significand);
-    }
-    len = static_cast<int>(digits.size() - n);
-    std::memcpy(buf, digits.data() + n, static_cast<std::size_t>(len));
-    decimal_exponent = d.exponent;
-}
-
 /*!
 @brief appends a decimal representation of e to buf
 @return a pointer to the element following the exponent.
@@ -1423,53 +1341,30 @@ inline char* write_shortest(char* first, const zmij::shortest_decimal d) noexcep
     return end + (three ? 5 : 4);
 }
 
-/// the powers of ten up to 10^16
-inline const std::array<std::uint64_t, 17>& powers_of_ten_16() noexcept
-{
-    static const std::array<std::uint64_t, 17> powers =
-    {
-        {
-            1u, 10u, 100u, 1000u, 10000u, 100000u, 1000000u, 10000000u, 100000000u, 1000000000u, 10000000000u,
-            100000000000u, 1000000000000u, 10000000000000u, 100000000000000u, 1000000000000000u, 10000000000000000u
-        }
-    };
-    return powers;
-}
-
 /*!
-@brief digits * 10^exp, as write_decimal() writes it, for the digits of a
+@brief whether FloatType is an IEEE 754 binary64 type (a double, or a long double
-double that need no conversion (count digits, at most 15, the first not 0;
+that has the same format, as with MSVC and on Apple's Arm CPUs)
-trailing zeros allowed): extended to 16 digits and written by write_shortest()
 
-@return a pointer past the text; up to 41 bytes at @a first are written
+These are the types the conversion of Zmij (see zmij.hpp) is used for; all
-        (some beyond the returned end)
+others (binary32, or a format the library does not know) use Grisu2.
 */
-JSON_HEDLEY_NON_NULL(1)
+template<typename FloatType>
-JSON_HEDLEY_RETURNS_NON_NULL
+constexpr bool has_binary64_format() noexcept
-inline char* write_short_decimal(char* first, std::uint64_t digits, int count, int exp) noexcept
 {
-    JSON_ASSERT(digits >= powers_of_ten_16()[static_cast<std::size_t>(count - 1)] && count <= 15);
+    return std::numeric_limits<FloatType>::is_iec559
-    const int scale = 16 - count;
+           && std::numeric_limits<FloatType>::digits == 53
-    return write_shortest(first, zmij::shortest_decimal{digits * powers_of_ten_16()[static_cast<std::size_t>(scale)], exp - scale - 1, 0, false});
+           && std::numeric_limits<FloatType>::max_exponent == 1024
+           && sizeof(FloatType) == sizeof(std::uint64_t);
 }
 
-/// as write_short_decimal(), counting the digits (not 0, less than 10^15)
+template<typename FloatType>
-JSON_HEDLEY_NON_NULL(1)
+struct is_binary64 : std::integral_constant<bool, has_binary64_format<FloatType>()> {};
-JSON_HEDLEY_RETURNS_NON_NULL
-inline char* write_short_decimal(char* first, std::uint64_t digits, int exp) noexcept
-{
-    JSON_ASSERT(digits != 0 && digits < 1000000000000000u);
-    // floor(log10(2^bits)) + 1 digits, or one less
-    const int log2_bound = ((64 - count_leading_zeros(digits)) * 1233) >> 12;
-    const int count = log2_bound + (digits >= powers_of_ten_16()[static_cast<std::size_t>(log2_bound)] ? 1 : 0);
-    return write_short_decimal(first, digits, count, exp);
-}
 
-/// a positive finite float (other than double): Grisu2 and format_buffer()
+/// a positive finite float (other than binary64): Grisu2 and format_buffer()
 template<typename FloatType>
 JSON_HEDLEY_NON_NULL(1, 2)
 JSON_HEDLEY_RETURNS_NON_NULL
-char* write_positive(char* first, const char* last, FloatType value)
+char* write_positive_grisu2(char* first, const char* last, FloatType value)
 {
     JSON_ASSERT(last - first >= std::numeric_limits<FloatType>::max_digits10);
     static_cast<void>(last); // (only used in the assertion)
@@ -1480,7 +1375,7 @@ char* write_positive(char* first, const char* last, FloatType value)
     // len is the length of the buffer, i.e., the number of decimal digits.
     int len = 0;
     int decimal_exponent = 0;
-    shortest_digits(first, len, decimal_exponent, value);
+    grisu2(first, len, decimal_exponent, value);
 
     JSON_ASSERT(len <= std::numeric_limits<FloatType>::max_digits10);
 
@@ -1496,15 +1391,16 @@ char* write_positive(char* first, const char* last, FloatType value)
     return format_buffer(first, len, decimal_exponent, kMinExp, kMaxExp);
 }
 
-/// a positive finite double: the shortest digits (Zmij), laid out by
+/// a positive finite binary64 number: the shortest digits (Zmij), laid out by
 /// write_shortest() (through a local buffer if [first, last) is shorter than
 /// the 41 bytes it may write)
+template<typename FloatType>
 JSON_HEDLEY_NON_NULL(1, 2)
 JSON_HEDLEY_RETURNS_NON_NULL
-inline char* write_positive(char* first, const char* last, double value)
+char* write_positive_zmij(char* first, const char* last, FloatType value)
 {
-    static_assert(std::numeric_limits<double>::is_iec559 && std::numeric_limits<double>::digits == 53,
+    static_assert(is_binary64<FloatType>::value,
-                  "internal error: the conversion of Zmij needs IEEE 754 binary64 doubles");
+                  "internal error: the conversion of Zmij needs IEEE 754 binary64 numbers");
     std::uint64_t bits = 0;
     std::memcpy(&bits, &value, sizeof(bits));
     const zmij::shortest_decimal d = zmij::to_shortest(bits);
@@ -1519,6 +1415,34 @@ inline char* write_positive(char* first, const char* last, double value)
     return first + len;
 }
 
+/// a positive finite binary64 number: Zmij (as a long double has the format of
+/// a double here, its bits are those of the double of the same value)
+template<typename FloatType>
+JSON_HEDLEY_NON_NULL(1, 2)
+JSON_HEDLEY_RETURNS_NON_NULL
+char* write_positive(char* first, const char* last, FloatType value, std::true_type /*is_binary64*/)
+{
+    return write_positive_zmij(first, last, value);
+}
+
+/// a positive finite float of any other format: Grisu2
+template<typename FloatType>
+JSON_HEDLEY_NON_NULL(1, 2)
+JSON_HEDLEY_RETURNS_NON_NULL
+char* write_positive(char* first, const char* last, FloatType value, std::false_type /*is_binary64*/)
+{
+    return write_positive_grisu2(first, last, value);
+}
+
+/// a positive finite float: Zmij for binary64 numbers, Grisu2 otherwise
+template<typename FloatType>
+JSON_HEDLEY_NON_NULL(1, 2)
+JSON_HEDLEY_RETURNS_NON_NULL
+char* write_positive(char* first, const char* last, FloatType value)
+{
+    return write_positive(first, last, value, is_binary64<FloatType> {});
+}
+
 }  // namespace dtoa_impl
 
 /*!

include/nlohmann/detail/conversions/zmij.hpp

@@ -36,13 +36,6 @@ computed from the compressed tables of Zmij beyond it.
 namespace zmij
 {
 
-/// significand * 10^exponent
-struct decimal
-{
-    std::uint64_t significand;
-    int exponent;
-};
-
 /// the compressed powers of ten of Zmij
 inline const std::array<std::uint64_t, 28>& pow10_minor() noexcept
 {
@@ -221,18 +214,6 @@ JSON_HEDLEY_ALWAYS_INLINE shortest_decimal to_shortest(std::uint64_t bits) noexc
     return shortest_decimal{integral, dec_exp, static_cast<unsigned char>(digit), !round_up && !round_down};
 }
 
-/// The shortest decimal in the rounding interval of a positive finite double
-/// given by its bits, as one number. The significand can end in zeros.
-inline decimal to_decimal(std::uint64_t bits) noexcept
-{
-    const shortest_decimal d = to_shortest(bits);
-    if (d.has_digit)
-    {
-        return decimal{(d.integral * 10) + d.digit, d.exponent};
-    }
-    return decimal{d.integral, d.exponent + 1};
-}
-
 }  // namespace zmij
 }  // namespace detail
 NLOHMANN_JSON_NAMESPACE_END

tests/src/unit-to_chars.cpp

@@ -15,6 +15,7 @@
 #include <nlohmann/json.hpp>
 using nlohmann::detail::dtoa_impl::reinterpret_bits;
 
+#include <algorithm>
 #include <array>
 #include <cmath>
 #include <cstdint>
@@ -666,13 +667,24 @@ void check_shortest(double v)
     const std::string text(buf.data(), end);
     CAPTURE(text)
     CHECK(parse_double(text) == v);
-    // the layout is that of format_buffer() for the same digits
+    // the layout is that of format_buffer() for the digits of Zmij
+    const auto sd = nlohmann::detail::zmij::to_shortest(reinterpret_bits<std::uint64_t>(v));
+    const std::uint64_t significand = sd.has_digit ? (sd.integral * 10) + sd.digit : sd.integral;
+    int exponent = sd.has_digit ? sd.exponent : sd.exponent + 1;
+    std::string significand_digits = std::to_string(significand);
+    while (significand_digits.size() > 1 && significand_digits.back() == '0')
+    {
+        significand_digits.pop_back();
+        ++exponent;
+    }
     std::array<char, 64> reference{};
-    int len = 0;
+    std::copy(significand_digits.begin(), significand_digits.end(), reference.begin());
-    int exponent = 0;
+    const char* const reference_end = nlohmann::detail::dtoa_impl::format_buffer(reference.data(), static_cast<int>(significand_digits.size()), exponent, -4, 15);
-    nlohmann::detail::dtoa_impl::shortest_digits(reference.data(), len, exponent, v);
-    const char* const reference_end = nlohmann::detail::dtoa_impl::format_buffer(reference.data(), len, exponent, -4, 15);
     CHECK(text == std::string(reference.data(), static_cast<std::size_t>(reference_end - reference.data())));
+    // and write_positive() is what to_chars() calls
+    std::array<char, 64> positive{};
+    const char* const positive_end = nlohmann::detail::dtoa_impl::write_positive(positive.data(), positive.data() + positive.size(), v);
+    CHECK(text == std::string(positive.data(), static_cast<std::size_t>(positive_end - positive.data())));
     const auto de = digits_and_exponent(text);
     const std::string& digits = de.first;
     if (digits.size() > 1)
@@ -785,3 +797,58 @@ TEST_CASE("shortest digits of doubles")
         }
     }
 }
+
+TEST_CASE("choice of the conversion")
+{
+    using nlohmann::detail::dtoa_impl::is_binary64;
+
+    SECTION("by the format of the type")
+    {
+        // Zmij needs binary64 numbers; everything else uses Grisu2
+        static_assert(!is_binary64<float>::value, "float is not binary64");
+        static_assert(is_binary64<double>::value == (std::numeric_limits<double>::is_iec559 && std::numeric_limits<double>::digits == 53),
+                      "double is binary64 where it is IEEE 754 with 53 digits");
+        static_assert(!is_binary64<int>::value, "integers are not binary64");
+        static_assert(is_binary64<long double>::value == (std::numeric_limits<long double>::is_iec559 && std::numeric_limits<long double>::digits == 53 && sizeof(long double) == 8),
+                      "long double is binary64 where it has the format of a double");
+        CHECK(!is_binary64<float>::value);
+        CHECK(is_binary64<double>::value);
+    }
+
+    SECTION("float: Grisu2, double: Zmij")
+    {
+        // 5.3165205877497296e+16 is one of the doubles for which Grisu2 does not find the shortest digits
+        constexpr double value = 5.3165205877497296e+16;
+        std::array<char, 64> buf{};
+        const char* const last = buf.data() + buf.size();
+
+        char* end = nlohmann::detail::dtoa_impl::write_positive(buf.data(), last, value);
+        CHECK(std::string(buf.data(), end) == "5.31652058774973e+16");
+        end = nlohmann::detail::dtoa_impl::write_positive_grisu2(buf.data(), last, value);
+        CHECK(std::string(buf.data(), end) == "5.3165205877497296e+16");
+
+        constexpr float f = 1.1754944e-38f;
+        end = nlohmann::detail::dtoa_impl::write_positive(buf.data(), last, f);
+        const std::string dispatched(buf.data(), end);
+        end = nlohmann::detail::dtoa_impl::write_positive_grisu2(buf.data(), last, f);
+        CHECK(dispatched == std::string(buf.data(), end));
+    }
+
+    SECTION("long double with the format of a double: Zmij")
+    {
+        // (on platforms where long double is wider, Grisu2 does not apply either: the snprintf fallback does)
+        if (std::numeric_limits<long double>::digits == 53 && std::numeric_limits<long double>::is_iec559)
+        {
+            using long_double_json = nlohmann::json::with_float_t<long double>;
+            for (const double d :
+                    {
+                        5.3165205877497296e+16, 1.0, 0.1, 123456.789, 2.2250738585072014e-308, 1.7976931348623157e+308, -5.3165205877497296e+16
+                    })
+            {
+                CAPTURE(d)
+                CHECK(long_double_json(static_cast<long double>(d)).dump() == nlohmann::json(d).dump());
+            }
+            CHECK(long_double_json(5.3165205877497296e+16L).dump() == "5.31652058774973e+16");
+        }
+    }
+}