Merge branch 'develop' into bench/json-view-comparison-fairness

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

tests/benchmarks/src/benchmarks.cpp

@@ -21,7 +21,7 @@
 #include <vector>
 #include <test_data.hpp>
 
-using json = nlohmann::json;
+using nlohmann::json;
 
 //////////////////////////////////////////////////////////////////////////////
 // parse JSON from file

tests/benchmarks/src/benchmarks_view.cpp

@@ -17,8 +17,8 @@
 #include <string>
 #include <test_data.hpp>
 
-using json = nlohmann::json;
-using json_document = nlohmann::json_document;
+using nlohmann::json;
+using nlohmann::json_document;
 
 static std::string read_file(const char* filename)
 {

tests/src/fuzzer-json_view_image.cpp

@@ -33,8 +33,8 @@ drivers.
     #error "the fuzzer drivers must be built without NDEBUG"
 #endif
 
-using json = nlohmann::json;
-using json_document = nlohmann::json_document;
+using nlohmann::json;
+using nlohmann::json_document;
 using image_check = json_document::image_check;
 
 // see http://llvm.org/docs/LibFuzzer.html

tests/src/fuzzer-parse_json_view.cpp

@@ -42,8 +42,8 @@ drivers.
     #error "the fuzzer drivers must be built without NDEBUG"
 #endif
 
-using json = nlohmann::json;
-using json_document = nlohmann::json_document;
+using nlohmann::json;
+using nlohmann::json_document;
 
 namespace
 {

tests/src/unit-bjdata.cpp

@@ -19,7 +19,7 @@
 
 #include <nlohmann/json.hpp>
 using nlohmann::json;
-using ordered_json = nlohmann::ordered_json;
+using nlohmann::ordered_json;
 
 #include <algorithm>
 #include <climits>

tests/src/unit-class_lexer.cpp

@@ -2042,3 +2042,118 @@ TEST_CASE("string scanning kernels")
     CHECK(nlohmann::detail::read_eight_bytes(bytes.data() + 8) == 0xFF0F0E0D0C0B0A09u);
     CHECK(nlohmann::detail::read_eight_bytes(reinterpret_cast<const char*>(bytes.data()) + 3) == 0x0B0A090807060504u);
 }
+
+TEST_CASE("find_ascii_copyable_run agrees with is_ascii_copyable for every byte")
+{
+    // the SWAR mask in find_ascii_copyable_run() encodes the predicate a
+    // second time; check every byte value at every position of a 24-byte buffer
+    // (three words) and for several lengths, with the byte at the end or inside
+    for (unsigned int value = 0; value < 256; ++value)
+    {
+        for (std::size_t position = 0; position < 24; ++position)
+        {
+            std::array<unsigned char, 24> buffer{};
+            buffer.fill(static_cast<unsigned char>('a'));
+            buffer[position] = static_cast<unsigned char>(value);
+            const bool copyable = nlohmann::detail::is_ascii_copyable(static_cast<unsigned char>(value));
+            for (const std::size_t n :
+                    {
+                        position, position + 1, std::size_t{24}
+                    })
+            {
+                if (n > 24)
+                {
+                    continue;
+                }
+                const std::size_t expected = (copyable || n <= position) ? n : position;
+                CAPTURE(value)
+                CAPTURE(position)
+                CAPTURE(n)
+                CHECK(nlohmann::detail::find_ascii_copyable_run(buffer.data(), n) == expected);
+            }
+        }
+    }
+}
+
+TEST_CASE("UTF-8 validators accept the same sequences")
+{
+    // The library checks UTF-8 in several places: detail::decode (serializer),
+    // the per-lead-byte switch of the lexer (json::accept), validate_one_utf8
+    // (bulk string scan), utf8_shape, and the byte path of the BON8 reader.
+    // They must accept exactly the same sequences (RFC 3629, section 4).
+    using nlohmann::json;
+
+    // bytes for the third and fourth position: the first two pairs are checked
+    // against all validators, the others only against the cheap ones
+    const std::array<std::array<unsigned char, 2>, 6> tails =
+    {
+        {
+            {{0x80, 0x80}}, {{0xBF, 0xBF}}, {{0x7F, 0x80}}, {{0xC0, 0x80}}, {{0x80, 0x7F}}, {{0x80, 0xC0}}
+        }
+    };
+
+    for (unsigned lead = 0x80; lead <= 0xFF; ++lead)
+    {
+        const auto shape = nlohmann::detail::utf8_shape(static_cast<std::uint8_t>(lead));
+        for (unsigned second = 0; second <= 0xFF; ++second)
+        {
+            for (std::size_t t = 0; t < tails.size(); ++t)
+            {
+                const std::array<unsigned char, 4> bytes = {{static_cast<unsigned char>(lead), static_cast<unsigned char>(second), tails[t][0], tails[t][1]}};
+
+                // expectation from the shape: the sequence is bytes[0..length)
+                const std::size_t length = shape.length;
+                bool expected = length >= 2 && second >= shape.second_lo && second <= shape.second_hi;
+                for (std::size_t i = 2; expected && i < length; ++i)
+                {
+                    expected = bytes[i] >= 0x80 && bytes[i] <= 0xBF;
+                }
+
+                // the DFA of the serializer
+                bool dfa = length >= 2;
+                if (dfa)
+                {
+                    std::uint8_t state = nlohmann::detail::UTF8_ACCEPT;
+                    std::uint32_t codep = 0;
+                    for (std::size_t i = 0; i < length; ++i)
+                    {
+                        nlohmann::detail::decode(state, codep, bytes[i]);
+                    }
+                    dfa = state == nlohmann::detail::UTF8_ACCEPT;
+                }
+
+                // the bulk path: exactly the sequence, and one byte short
+                const std::size_t bulk = nlohmann::detail::validate_one_utf8(bytes.data(), length);
+                const std::size_t bulk_short = length > 0 ? nlohmann::detail::validate_one_utf8(bytes.data(), length - 1) : 0;
+
+                CAPTURE(lead)
+                CAPTURE(second)
+                CAPTURE(t)
+                CHECK((dfa == expected && bulk == (expected ? length : 0) && bulk_short == 0));
+
+                if (t < 2 && length >= 2)
+                {
+                    // the lexer's byte path
+                    const std::string sequence(reinterpret_cast<const char*>(bytes.data()), length); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast)
+                    CHECK(json::accept("\"" + sequence + "\"") == expected);
+
+                    // the byte path of the BON8 reader (a string ends at 0xFF)
+                    if (lead >= 0xC2 && lead <= 0xF7)
+                    {
+                        std::vector<std::uint8_t> packed(bytes.begin(), bytes.begin() + static_cast<std::ptrdiff_t>(length));
+                        packed.push_back(0xFF);
+                        const json j = json::from_bon8(packed, true, false);
+                        if (expected)
+                        {
+                            CHECK(j == json(sequence));
+                        }
+                        else
+                        {
+                            CHECK_FALSE((j.is_string() && j.get<std::string>() == sequence));
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

tests/src/unit-constructor1.cpp

@@ -361,6 +361,40 @@ TEST_CASE("constructors")
             CHECK_THROWS_WITH_AS((j.get<std::pair<int, int>>()), "[json.exception.out_of_range.401] array index 1 is out of range", json::out_of_range&);
             CHECK_THROWS_WITH_AS((j.get<std::tuple<int, int>>()), "[json.exception.out_of_range.401] array index 1 is out of range", json::out_of_range&);
             CHECK_THROWS_WITH_AS((j.get<std::array<int, 3>>()), "[json.exception.out_of_range.401] array index 1 is out of range", json::out_of_range&);
+#if JSON_DIAGNOSTICS
+            // the exception names the element that has the wrong type
+            CHECK_THROWS_WITH_AS((j.get<std::array<std::string, 3>>()), "[json.exception.type_error.302] (/0) type must be string, but is number", json::type_error&);
+#else
+            CHECK_THROWS_WITH_AS((j.get<std::array<std::string, 3>>()), "[json.exception.type_error.302] type must be string, but is number", json::type_error&);
+#endif
+
+            // the elements are converted in order, so a type error comes before a missing element
+            json const j2{1, "two"};
+            std::array<int, 3> a{{0, 0, 0}};
+#if JSON_DIAGNOSTICS
+            CHECK_THROWS_WITH_AS((j2.get<std::array<int, 3>>()), "[json.exception.type_error.302] (/1) type must be number, but is string", json::type_error&);
+            CHECK_THROWS_WITH_AS(j2.get_to(a), "[json.exception.type_error.302] (/1) type must be number, but is string", json::type_error&);
+#else
+            CHECK_THROWS_WITH_AS((j2.get<std::array<int, 3>>()), "[json.exception.type_error.302] type must be number, but is string", json::type_error&);
+            CHECK_THROWS_WITH_AS(j2.get_to(a), "[json.exception.type_error.302] type must be number, but is string", json::type_error&);
+#endif
+            CHECK_THROWS_WITH_AS(j.get_to(a), "[json.exception.out_of_range.401] array index 1 is out of range", json::out_of_range&);
+#if !defined(JSON_NOEXCEPTION) // without exceptions, the get_to calls in CHECK_THROWS_WITH_AS do not run
+            CHECK(a[0] == 1);
+#endif
+
+            json const j3{"one", "two"};
+            CHECK_THROWS_WITH_AS((j3.get<std::array<std::string, 3>>()), "[json.exception.out_of_range.401] array index 2 is out of range", json::out_of_range&);
+        }
+
+        SECTION("std::array from a longer array or from booleans")
+        {
+            json const j{1, 2.5, true, 4u};
+            CHECK(j.get<std::array<int, 3>>() == (std::array<int, 3> {{1, 2, 1}}));
+            std::array<int, 4> a{{0, 0, 0, 0}};
+            j.get_to(a);
+            CHECK(a == (std::array<int, 4> {{1, 2, 1, 4}}));
+            CHECK(j.get<std::array<int, 0>>().empty());
         }
 
         SECTION("std::forward_list<json>")

tests/src/unit-diagnostic-positions.cpp

@@ -14,7 +14,7 @@
 #define JSON_DIAGNOSTIC_POSITIONS 1
 #include <nlohmann/json.hpp>
 
-using json = nlohmann::json;
+using nlohmann::json;
 
 TEST_CASE("Better diagnostics with positions")
 {

tests/src/unit-disabled_exceptions.cpp

@@ -13,7 +13,7 @@ DOCTEST_GCC_SUPPRESS_WARNING_PUSH
 DOCTEST_GCC_SUPPRESS_WARNING("-Wnoexcept")
 
 #include <nlohmann/json.hpp>
-using json = nlohmann::json;
+using nlohmann::json;
 #ifdef JSON_TEST_NO_GLOBAL_UDLS
     using namespace nlohmann::literals; // NOLINT(google-build-using-namespace)
 #endif

tests/src/unit-element_access2.cpp

@@ -1948,7 +1948,7 @@ TEST_CASE_TEMPLATE("element access 2 (additional value() tests)", Json, nlohmann
 #ifdef JSON_HAS_CPP_17
 TEST_CASE("operator[] with user-defined std::string_view-convertible types")
 {
-    using json = nlohmann::json;
+    using nlohmann::json;
 
     class TestClass
     {
@@ -2031,7 +2031,7 @@ TEST_CASE("keys convertible to std::string_view work with all lookup functions (
 
     SECTION("nlohmann::json")
     {
-        using json = nlohmann::json;
+        using nlohmann::json;
 
         SECTION("ViewKey")
         {
@@ -2072,7 +2072,7 @@ TEST_CASE("keys convertible to std::string_view work with all lookup functions (
 
     SECTION("nlohmann::ordered_json")
     {
-        using ordered_json = nlohmann::ordered_json;
+        using nlohmann::ordered_json;
 
         SECTION("ViewKey")
         {

tests/src/unit-explicit_instantiation.cpp

@@ -15,7 +15,7 @@
 #include "doctest_compatibility.h"
 
 #include <nlohmann/json.hpp>
-using json = nlohmann::json;
+using nlohmann::json;
 
 /////////////////////////////////////////////////////////////////////
 // for #4825 - explicitly instantiating basic_json must compile; this

tests/src/unit-format-as.cpp

@@ -9,8 +9,8 @@
 #include "doctest_compatibility.h"
 
 #include <nlohmann/json.hpp>
-using json = nlohmann::json;
-using ordered_json = nlohmann::ordered_json;
+using nlohmann::json;
+using nlohmann::ordered_json;
 
 namespace
 {

tests/src/unit-hash.cpp

@@ -9,8 +9,8 @@
 #include "doctest_compatibility.h"
 
 #include <nlohmann/json.hpp>
-using json = nlohmann::json;
-using ordered_json = nlohmann::ordered_json;
+using nlohmann::json;
+using nlohmann::ordered_json;
 
 #include <limits>
 #include <set>

tests/src/unit-no_automatic_udls.cpp

@@ -19,7 +19,7 @@
 #include <utility>
 
 #include <nlohmann/json.hpp>
-using json = nlohmann::json;
+using nlohmann::json;
 
 // An argument type whose associated namespace is the library namespace, so
 // argument-dependent lookup of a literal operator called by its function name

tests/src/unit-no_io_and_user_exceptions.cpp

@@ -38,7 +38,7 @@ static int json_throw_user_call_count = 0; // NOLINT(cppcoreguidelines-avoid-non
 #include "doctest_compatibility.h"
 
 #include <nlohmann/json.hpp>
-using json = nlohmann::json;
+using nlohmann::json;
 
 TEST_CASE("JSON_NO_IO")
 {

tests/src/unit-regression2.cpp

@@ -33,8 +33,8 @@
 
 #define JSON_TESTS_PRIVATE
 #include <nlohmann/json.hpp>
-using json = nlohmann::json;
-using ordered_json = nlohmann::ordered_json;
+using nlohmann::json;
+using nlohmann::ordered_json;
 #ifdef JSON_TEST_NO_GLOBAL_UDLS
     using namespace nlohmann::literals; // NOLINT(google-build-using-namespace)
 #endif

tests/src/unit-regression3.cpp

@@ -36,8 +36,8 @@
 
 #define JSON_TESTS_PRIVATE
 #include <nlohmann/json.hpp>
-using json = nlohmann::json;
-using ordered_json = nlohmann::ordered_json;
+using nlohmann::json;
+using nlohmann::ordered_json;
 #ifdef JSON_TEST_NO_GLOBAL_UDLS
     using namespace nlohmann::literals; // NOLINT(google-build-using-namespace)
 #endif

tests/src/unit-std-format.cpp

@@ -16,8 +16,8 @@
 #include "doctest_compatibility.h"
 
 #include <nlohmann/json.hpp>
-using json = nlohmann::json;
-using ordered_json = nlohmann::ordered_json;
+using nlohmann::json;
+using nlohmann::ordered_json;
 
 // JSON_HAS_CPP_20 (do not remove; see note at top of file)
 #if JSON_HAS_STD_FORMAT