Split the Visual Studio 2019 AppVeyor job into two shards

The Visual Studio 2019 job (Release, x86, JSON_ImplicitConversions=OFF)
builds the tests for all standards and takes 49 to 57 minutes when it
passes, close to AppVeyor's 60-minute limit per job. In build 54917118
(#5867), build 54917312 (#5620), and build 54930878 (#5851) it was
stopped at the limit while still compiling. Split it with JSON_TestShard
like the Visual Studio 2017 jobs.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

.github/external_ci/appveyor.yml

@@ -18,7 +18,8 @@ environment:
   matrix:
     # The Visual Studio 2017 jobs compile everything with /std:c++17, so they
     # only build the C++17 variant of each test, split into two jobs each to
-    # stay below AppVeyor's 60-minute limit per job.
+    # stay below AppVeyor's 60-minute limit per job. The Visual Studio 2019
+    # job builds the tests for all standards and is split for the same reason.
     - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
       configuration: Debug
       platform: x86
@@ -50,7 +51,13 @@ environment:
       configuration: Release
       platform: x86
       CXX_FLAGS: "/W4 /WX"
-      CMAKE_OPTIONS: "-DJSON_ImplicitConversions=OFF"
+      CMAKE_OPTIONS: "-DJSON_ImplicitConversions=OFF -DJSON_TestShard=0/2"
+      GENERATOR: Visual Studio 16 2019
+    - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
+      configuration: Release
+      platform: x86
+      CXX_FLAGS: "/W4 /WX"
+      CMAKE_OPTIONS: "-DJSON_ImplicitConversions=OFF -DJSON_TestShard=1/2"
       GENERATOR: Visual Studio 16 2019
 
     - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015

include/nlohmann/detail/input/lexer.hpp

@@ -1625,12 +1625,9 @@ scan_number_done:
         const std::size_t avail = rem + 1;
 
         // validate + classify the number extent (mirrors scan_number()'s grammar)
-        // and accumulate the mantissa digits on the way, so that tokens with up
-        // to 19 digits need no second pass for their value
         std::size_t i = 0;
         std::size_t dot_index = std::string::npos;
         token_type number_type = token_type::value_unsigned;
-        std::uint64_t w = 0;
         if (data[0] == '-')
         {
             number_type = token_type::value_integer;
@@ -1640,21 +1637,22 @@ scan_number_done:
                 return token_type::uninitialized;
             }
         }
-        const std::size_t integer_begin = i;
         if (data[i] == '0')
         {
             ++i;
         }
         else if (data[i] >= '1' && data[i] <= '9')
         {
-            i = accumulate_digits(data, i, avail, w);
+            ++i;
+            while (i < avail && data[i] >= '0' && data[i] <= '9')
+            {
+                ++i;
+            }
         }
         else
         {
             return token_type::uninitialized;
         }
-        std::size_t digit_count = i - integer_begin;
-        std::size_t fraction_digits = 0;
         if (i < avail && data[i] == '.')
         {
             number_type = token_type::value_float;
@@ -1664,39 +1662,29 @@ scan_number_done:
             {
                 return token_type::uninitialized;
             }
-            const std::size_t fraction_begin = i;
-            i = accumulate_digits(data, i, avail, w);
-            fraction_digits = i - fraction_begin;
-            digit_count += fraction_digits;
+            while (i < avail && data[i] >= '0' && data[i] <= '9')
+            {
+                ++i;
+            }
         }
         // the mantissa ends here, whether or not an exponent part follows
         const std::size_t mantissa_end = i;
-        std::int64_t exponent = 0;
         if (i < avail && (data[i] == 'e' || data[i] == 'E'))
         {
             number_type = token_type::value_float;
             ++i;
-            bool negative_exponent = false;
             if (i < avail && (data[i] == '+' || data[i] == '-'))
             {
-                negative_exponent = data[i] == '-';
                 ++i;
             }
             if (i >= avail || !(data[i] >= '0' && data[i] <= '9'))
             {
                 return token_type::uninitialized;
             }
-            // saturated far beyond every range, see parse_float_exponent()
-            constexpr std::int64_t saturation = 100000000000000000; // 10^17
             while (i < avail && data[i] >= '0' && data[i] <= '9')
             {
-                if (exponent < saturation)
-                {
-                    exponent = (exponent * 10) + (data[i] - '0');
-                }
                 ++i;
             }
-            exponent = negative_exponent ? -exponent : exponent;
         }
         const std::size_t len = i;
         if (number_may_continue(len == avail, std::integral_constant<bool, bulk_window> {}))
@@ -1717,10 +1705,7 @@ scan_number_done:
 #if !JSON_DIAGNOSTIC_POSITIONS
         if (number_type != token_type::value_float)
         {
-            // up to 18 digits always fit into std::uint64_t and std::int64_t
-            const token_type integer_result = (digit_count <= 18)
-                                              ? store_integer(number_type, w)
-                                              : convert_integer(number_type, data, data + len);
+            const token_type integer_result = convert_integer(number_type, data, data + len);
             if (JSON_HEDLEY_LIKELY(integer_result != token_type::uninitialized))
             {
                 bulk_consume(data + 1, len - 1);
@@ -1743,20 +1728,6 @@ scan_number_done:
         bulk_consume(data + 1, len - 1);
         read_number_terminator(std::integral_constant<bool, bulk_window> {});
 
-        // a float whose digits all fit into w converts from the accumulated
-        // value; longer ones (and other float formats) take the general path
-        if (number_type == token_type::value_float && digit_count <= 19)
-        {
-            float_significand s;
-            s.w = w;
-            s.exponent = exponent - static_cast<std::int64_t>(fraction_digits);
-            s.negative = data[0] == '-';
-            if (convert_accumulated_float(s, std::integral_constant<bool, has_native_float_format<number_float_t>::value> {}))
-            {
-                return token_type::value_float;
-            }
-        }
-
         return convert_number(number_type, mantissa_end);
     }
 
@@ -1785,35 +1756,6 @@ scan_number_done:
     /// the input is not handed back, so the terminator can stay unread
     void read_number_terminator(std::false_type /*window*/) const noexcept {}
 
-    /// binary32/binary64: the value of a float token from its accumulated
-    /// significand (see scan_number_bulk_contiguous())
-    bool convert_accumulated_float(const float_significand& s, std::true_type /*native*/) noexcept
-    {
-        value_float = decimal_to_float<number_float_t>(s);
-        return true;
-    }
-
-    /// other formats (e.g. x87 long double): not instantiated for them, as
-    /// decimal_to_float() only supports binary32/binary64
-    static bool convert_accumulated_float(const float_significand& /*s*/, std::false_type /*native*/) noexcept
-    {
-        return false;
-    }
-
-    /// the value of an integer token with at most 18 digits, accumulated in @a w
-    token_type store_integer(token_type number_type, std::uint64_t w) noexcept
-    {
-        if (number_type == token_type::value_unsigned)
-        {
-            value_unsigned = static_cast<number_unsigned_t>(w);
-            // reject values that do not round-trip into a narrower type
-            return (static_cast<std::uint64_t>(value_unsigned) == w) ? token_type::value_unsigned : token_type::uninitialized;
-        }
-        const std::int64_t x = -static_cast<std::int64_t>(w);
-        value_integer = static_cast<number_integer_t>(x);
-        return (static_cast<std::int64_t>(value_integer) == x) ? token_type::value_integer : token_type::uninitialized;
-    }
-
     /// contiguous input: try the number fast path, else the byte-path scanner
     token_type scan_number_dispatch(std::true_type /*bulk*/)
     {

include/nlohmann/detail/input/number_parse.hpp

@@ -275,53 +275,13 @@ JSON_HEDLEY_ALWAYS_INLINE std::uint32_t parse_eight_digits(std::uint64_t v) noex
     return static_cast<std::uint32_t>(((v & 0x0000FFFF0000FFFFu) * 42949672960001u) >> 32u);
 }
 
-/// whether the eight bytes in @a v (see read_eight_bytes()) are all ASCII
-/// digits (after fast_float)
-JSON_HEDLEY_ALWAYS_INLINE bool is_eight_digits(std::uint64_t v) noexcept
+/// whether [first, last) contains a digit other than '0'
+inline bool has_nonzero_digit(const char* first, const char* last) noexcept
 {
-    return ((v & 0xF0F0F0F0F0F0F0F0u) | (((v + 0x0606060606060606u) & 0xF0F0F0F0F0F0F0F0u) >> 4u)) == 0x3333333333333333u;
-}
-
-/*!
-@brief skip the ASCII digits starting at data[i] and accumulate their value
-
-Eight digits are taken at a time while at least eight bytes remain. @a w wraps
-around after 19 digits; the caller only uses it for shorter digit runs.
-
-@return the index of the first byte at or after @a i that is not a digit
-*/
-JSON_HEDLEY_ALWAYS_INLINE std::size_t accumulate_digits(const char* data, std::size_t i, std::size_t avail, std::uint64_t& w) noexcept
-{
-    while (avail - i >= 8)
+    return std::any_of(first, last, [](char c) noexcept
     {
-        const std::uint64_t v = read_eight_bytes(data + i);
-        if (!is_eight_digits(v))
-        {
-            break;
-        }
-        w = (w * 100000000u) + parse_eight_digits(v);
-        i += 8;
-    }
-    while (i < avail && data[i] >= '0' && data[i] <= '9')
-    {
-        w = (w * 10u) + static_cast<std::uint64_t>(data[i] - '0');
-        ++i;
-    }
-    return i;
-}
-
-/// whether [first, last) contains a digit other than '0'; a plain loop, as
-/// GCC calls std::any_of out of line even for the usual empty range
-JSON_HEDLEY_ALWAYS_INLINE bool has_nonzero_digit(const char* first, const char* last) noexcept
-{
-    for (; first != last; ++first)
-    {
-        if (*first != '0')
-        {
-            return true;
-        }
-    }
-    return false;
+        return c != '0';
+    });
 }
 
 /// the value of the validated exponent digits [+-]?[0-9]+ in [first, last),
@@ -367,7 +327,7 @@ truncated is set.
                                    std::string::npos if there is none
 @param[in] mantissa_end            index of the 'e'/'E', or the token length
 */
-JSON_HEDLEY_ALWAYS_INLINE float_significand parse_float_significand(const char* first, const char* last,
+inline float_significand parse_float_significand(const char* first, const char* last,
         std::size_t decimal_point_position, std::size_t mantissa_end) noexcept
 {
     float_significand s;

single_include/nlohmann/json.hpp

@@ -10361,53 +10361,13 @@ JSON_HEDLEY_ALWAYS_INLINE std::uint32_t parse_eight_digits(std::uint64_t v) noex
     return static_cast<std::uint32_t>(((v & 0x0000FFFF0000FFFFu) * 42949672960001u) >> 32u);
 }
 
-/// whether the eight bytes in @a v (see read_eight_bytes()) are all ASCII
-/// digits (after fast_float)
-JSON_HEDLEY_ALWAYS_INLINE bool is_eight_digits(std::uint64_t v) noexcept
+/// whether [first, last) contains a digit other than '0'
+inline bool has_nonzero_digit(const char* first, const char* last) noexcept
 {
-    return ((v & 0xF0F0F0F0F0F0F0F0u) | (((v + 0x0606060606060606u) & 0xF0F0F0F0F0F0F0F0u) >> 4u)) == 0x3333333333333333u;
-}
-
-/*!
-@brief skip the ASCII digits starting at data[i] and accumulate their value
-
-Eight digits are taken at a time while at least eight bytes remain. @a w wraps
-around after 19 digits; the caller only uses it for shorter digit runs.
-
-@return the index of the first byte at or after @a i that is not a digit
-*/
-JSON_HEDLEY_ALWAYS_INLINE std::size_t accumulate_digits(const char* data, std::size_t i, std::size_t avail, std::uint64_t& w) noexcept
-{
-    while (avail - i >= 8)
+    return std::any_of(first, last, [](char c) noexcept
     {
-        const std::uint64_t v = read_eight_bytes(data + i);
-        if (!is_eight_digits(v))
-        {
-            break;
-        }
-        w = (w * 100000000u) + parse_eight_digits(v);
-        i += 8;
-    }
-    while (i < avail && data[i] >= '0' && data[i] <= '9')
-    {
-        w = (w * 10u) + static_cast<std::uint64_t>(data[i] - '0');
-        ++i;
-    }
-    return i;
-}
-
-/// whether [first, last) contains a digit other than '0'; a plain loop, as
-/// GCC calls std::any_of out of line even for the usual empty range
-JSON_HEDLEY_ALWAYS_INLINE bool has_nonzero_digit(const char* first, const char* last) noexcept
-{
-    for (; first != last; ++first)
-    {
-        if (*first != '0')
-        {
-            return true;
-        }
-    }
-    return false;
+        return c != '0';
+    });
 }
 
 /// the value of the validated exponent digits [+-]?[0-9]+ in [first, last),
@@ -10453,7 +10413,7 @@ truncated is set.
                                    std::string::npos if there is none
 @param[in] mantissa_end            index of the 'e'/'E', or the token length
 */
-JSON_HEDLEY_ALWAYS_INLINE float_significand parse_float_significand(const char* first, const char* last,
+inline float_significand parse_float_significand(const char* first, const char* last,
         std::size_t decimal_point_position, std::size_t mantissa_end) noexcept
 {
     float_significand s;
@@ -13156,12 +13116,9 @@ scan_number_done:
         const std::size_t avail = rem + 1;
 
         // validate + classify the number extent (mirrors scan_number()'s grammar)
-        // and accumulate the mantissa digits on the way, so that tokens with up
-        // to 19 digits need no second pass for their value
         std::size_t i = 0;
         std::size_t dot_index = std::string::npos;
         token_type number_type = token_type::value_unsigned;
-        std::uint64_t w = 0;
         if (data[0] == '-')
         {
             number_type = token_type::value_integer;
@@ -13171,21 +13128,22 @@ scan_number_done:
                 return token_type::uninitialized;
             }
         }
-        const std::size_t integer_begin = i;
         if (data[i] == '0')
         {
             ++i;
         }
         else if (data[i] >= '1' && data[i] <= '9')
         {
-            i = accumulate_digits(data, i, avail, w);
+            ++i;
+            while (i < avail && data[i] >= '0' && data[i] <= '9')
+            {
+                ++i;
+            }
         }
         else
         {
             return token_type::uninitialized;
         }
-        std::size_t digit_count = i - integer_begin;
-        std::size_t fraction_digits = 0;
         if (i < avail && data[i] == '.')
         {
             number_type = token_type::value_float;
@@ -13195,39 +13153,29 @@ scan_number_done:
             {
                 return token_type::uninitialized;
             }
-            const std::size_t fraction_begin = i;
-            i = accumulate_digits(data, i, avail, w);
-            fraction_digits = i - fraction_begin;
-            digit_count += fraction_digits;
+            while (i < avail && data[i] >= '0' && data[i] <= '9')
+            {
+                ++i;
+            }
         }
         // the mantissa ends here, whether or not an exponent part follows
         const std::size_t mantissa_end = i;
-        std::int64_t exponent = 0;
         if (i < avail && (data[i] == 'e' || data[i] == 'E'))
         {
             number_type = token_type::value_float;
             ++i;
-            bool negative_exponent = false;
             if (i < avail && (data[i] == '+' || data[i] == '-'))
             {
-                negative_exponent = data[i] == '-';
                 ++i;
             }
             if (i >= avail || !(data[i] >= '0' && data[i] <= '9'))
             {
                 return token_type::uninitialized;
             }
-            // saturated far beyond every range, see parse_float_exponent()
-            constexpr std::int64_t saturation = 100000000000000000; // 10^17
             while (i < avail && data[i] >= '0' && data[i] <= '9')
             {
-                if (exponent < saturation)
-                {
-                    exponent = (exponent * 10) + (data[i] - '0');
-                }
                 ++i;
             }
-            exponent = negative_exponent ? -exponent : exponent;
         }
         const std::size_t len = i;
         if (number_may_continue(len == avail, std::integral_constant<bool, bulk_window> {}))
@@ -13248,10 +13196,7 @@ scan_number_done:
 #if !JSON_DIAGNOSTIC_POSITIONS
         if (number_type != token_type::value_float)
         {
-            // up to 18 digits always fit into std::uint64_t and std::int64_t
-            const token_type integer_result = (digit_count <= 18)
-                                              ? store_integer(number_type, w)
-                                              : convert_integer(number_type, data, data + len);
+            const token_type integer_result = convert_integer(number_type, data, data + len);
             if (JSON_HEDLEY_LIKELY(integer_result != token_type::uninitialized))
             {
                 bulk_consume(data + 1, len - 1);
@@ -13274,20 +13219,6 @@ scan_number_done:
         bulk_consume(data + 1, len - 1);
         read_number_terminator(std::integral_constant<bool, bulk_window> {});
 
-        // a float whose digits all fit into w converts from the accumulated
-        // value; longer ones (and other float formats) take the general path
-        if (number_type == token_type::value_float && digit_count <= 19)
-        {
-            float_significand s;
-            s.w = w;
-            s.exponent = exponent - static_cast<std::int64_t>(fraction_digits);
-            s.negative = data[0] == '-';
-            if (convert_accumulated_float(s, std::integral_constant<bool, has_native_float_format<number_float_t>::value> {}))
-            {
-                return token_type::value_float;
-            }
-        }
-
         return convert_number(number_type, mantissa_end);
     }
 
@@ -13316,35 +13247,6 @@ scan_number_done:
     /// the input is not handed back, so the terminator can stay unread
     void read_number_terminator(std::false_type /*window*/) const noexcept {}
 
-    /// binary32/binary64: the value of a float token from its accumulated
-    /// significand (see scan_number_bulk_contiguous())
-    bool convert_accumulated_float(const float_significand& s, std::true_type /*native*/) noexcept
-    {
-        value_float = decimal_to_float<number_float_t>(s);
-        return true;
-    }
-
-    /// other formats (e.g. x87 long double): not instantiated for them, as
-    /// decimal_to_float() only supports binary32/binary64
-    static bool convert_accumulated_float(const float_significand& /*s*/, std::false_type /*native*/) noexcept
-    {
-        return false;
-    }
-
-    /// the value of an integer token with at most 18 digits, accumulated in @a w
-    token_type store_integer(token_type number_type, std::uint64_t w) noexcept
-    {
-        if (number_type == token_type::value_unsigned)
-        {
-            value_unsigned = static_cast<number_unsigned_t>(w);
-            // reject values that do not round-trip into a narrower type
-            return (static_cast<std::uint64_t>(value_unsigned) == w) ? token_type::value_unsigned : token_type::uninitialized;
-        }
-        const std::int64_t x = -static_cast<std::int64_t>(w);
-        value_integer = static_cast<number_integer_t>(x);
-        return (static_cast<std::int64_t>(value_integer) == x) ? token_type::value_integer : token_type::uninitialized;
-    }
-
     /// contiguous input: try the number fast path, else the byte-path scanner
     token_type scan_number_dispatch(std::true_type /*bulk*/)
     {

tests/src/unit-class_lexer.cpp

@@ -15,12 +15,9 @@ using nlohmann::json;
 #include <array> // array
 #include <cstdint> // uint32_t, uint64_t
 #include <cstdio> // snprintf
-#include <cmath> // isfinite
 #include <cstdlib> // strtod
 #include <cstring> // memcpy
 #include <limits> // numeric_limits
-#include <initializer_list> // initializer_list
-#include <list> // list
 #include <map> // map
 #include <random> // mt19937
 #include <sstream> // stringstream
@@ -335,112 +332,6 @@ TEST_CASE("lexer number fast path")
         }
     }
 
-#if !defined(JSON_NOEXCEPTION)
-    // this section reports parse errors, which abort when exceptions are off
-    SECTION("digits accumulated while scanning")
-    {
-        // The fast path accumulates the digits while it validates them, eight
-        // at a time where possible, and converts tokens with at most 19
-        // digits (floats) or 18 digits (integers) from that value. Every digit
-        // count from 1 to 25, in the integer and in the fraction part, with
-        // and without sign and exponent, must give the byte path's result. A
-        // std::list has no contiguous storage, so parsing it takes the byte
-        // path.
-        // type and value, or the error (values beyond the range of double
-        // are rejected with out_of_range.406 on both paths)
-        const auto outcome = [](const std::string & doc, bool byte_path) -> std::string
-        {
-            JSON_TRY
-            {
-                const std::list<char> chars(doc.begin(), doc.end());
-                const json j = byte_path ? json::parse(chars.begin(), chars.end()) : json::parse(doc);
-                return std::string(j[0].type_name()) + '|' + j.dump();
-            }
-            JSON_CATCH (const json::exception & e)
-            {
-                return e.what();
-            }
-        };
-
-        // the concatenation of @a parts
-        const auto join = [](std::initializer_list<std::string> parts)
-        {
-            std::string result;
-            for (const auto& part : parts)
-            {
-                result += part;
-            }
-            return result;
-        };
-
-        std::vector<std::string> numbers;
-        for (std::size_t digits = 1; digits <= 25; ++digits)
-        {
-            std::string run;
-            for (std::size_t i = 0; i < digits; ++i)
-            {
-                run += static_cast<char>('1' + static_cast<char>((i * 7) % 9));
-            }
-            const std::string nines(digits, '9');
-            for (const char* sign_text :
-                    {"", "-"
-                    })
-            {
-                const std::string sign = sign_text;
-                numbers.push_back(join({sign, run}));
-                numbers.push_back(join({sign, nines}));
-                numbers.push_back(join({sign, "1", std::string(digits - 1, '0')}));
-                numbers.push_back(join({sign, "0.", run}));
-                numbers.push_back(join({sign, "0.", std::string(digits, '0'), "1"}));
-                numbers.push_back(join({sign, run, ".", run}));
-                numbers.push_back(join({sign, run, "e-", std::to_string(digits)}));
-                numbers.push_back(join({sign, "1.", run, "E+", std::to_string(digits * 13)}));
-            }
-        }
-        // exponents beyond every range, and zeros
-        numbers.insert(numbers.end(), {"1e400", "1e-400", "-1e99999999999999999999", "1e-99999999999999999999",
-                                       "0e99999999999999999999", "0.0e-5", "-0.000", "123456789e-0000000000000000000000330"
-                                      });
-
-        for (const auto& n : numbers)
-        {
-            CAPTURE(n)
-            const std::string doc = "[" + n + "]";
-            const std::string a = outcome(doc, false);
-            CHECK(a == outcome(doc, true));
-            const double expected = std::strtod(n.c_str(), nullptr);
-            if (a.compare(0, 6, "number") == 0 && std::isfinite(expected) && n.find_first_of(".eE") != std::string::npos)
-            {
-                CHECK(json::parse(doc)[0].get<double>() == expected);
-            }
-        }
-    }
-#endif
-
-    SECTION("integers that do not fit a narrower type become floats")
-    {
-        // up to 18 digits, the fast path stores the accumulated value without
-        // an overflow check; a narrower number_integer_t/number_unsigned_t
-        // must still see the value rejected and converted as a float
-        using json32 = nlohmann::basic_json<std::map, std::vector, std::string, bool, std::int32_t, std::uint32_t, float>;
-        for (const char* n :
-                {"4294967295", "4294967296", "2147483647", "-2147483648", "-2147483649", "99999999999", "-99999999999"
-                })
-        {
-            CAPTURE(n)
-            const std::string doc = std::string("[") + n + "]";
-            const std::list<char> chars(doc.begin(), doc.end());
-            const json32 a = json32::parse(doc);
-            const json32 b = json32::parse(chars.begin(), chars.end());
-            CHECK(a[0].type() == b[0].type());
-            CHECK(a == b);
-        }
-        CHECK(json32::parse("[4294967295]")[0].is_number_unsigned());
-        CHECK(json32::parse("[4294967296]")[0].is_number_float());
-        CHECK(json32::parse("[-2147483648]")[0].is_number_integer());
-        CHECK(json32::parse("[-2147483649]")[0].is_number_float());
-    }
-
     SECTION("token type classification")
     {
         CHECK((scan_string("0") == json::lexer::token_type::value_unsigned));