Merge branch 'json-view/03-string-scan' into json-view/04-unicode-escapes

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

tests/CMakeLists.txt

@@ -89,7 +89,8 @@ target_compile_options(test_main PUBLIC
     # https://github.com/nlohmann/json/pull/3229
     $<$<CXX_COMPILER_ID:Intel>:-diag-disable=2196>
 
-    $<$<CXX_COMPILER_ID:GNU>:-Wno-deprecated-declarations>
+    # several tests call deprecated functions on purpose to keep them covered
+    $<$<NOT:$<CXX_COMPILER_ID:MSVC>>:-Wno-deprecated-declarations>
     $<$<CXX_COMPILER_ID:Intel>:-diag-disable=1786>)
 target_include_directories(test_main SYSTEM PUBLIC
     thirdparty/doctest)
@@ -101,7 +102,7 @@ target_link_libraries(test_main PUBLIC ${NLOHMANN_JSON_TARGET_NAME})
 # test-regression1 needs it on its include path (see json_test_set_test_options
 # below), rather than every test-* target via test_main.
 add_library(fifo_map_include INTERFACE)
-target_include_directories(fifo_map_include INTERFACE thirdparty/fifo_map)
+target_include_directories(fifo_map_include SYSTEM INTERFACE thirdparty/fifo_map)
 
 #############################################################################
 # define test- and standard-specific build settings

tests/src/unit-allocator.cpp

@@ -372,6 +372,11 @@ void check_deep_copy_survives_failing_allocation(bool nest_objects)
 
 TEST_CASE("copy of a deeply nested value survives a failing allocation (#5640)")
 {
+    // With iterator debugging (MSVC STL debug builds, also used by clang-cl),
+    // containers allocate a debug proxy through the allocator inside their
+    // noexcept move constructors, so failing that allocation terminates the
+    // program instead of throwing std::bad_alloc. Nothing to check there.
+#if !(defined(_ITERATOR_DEBUG_LEVEL) && _ITERATOR_DEBUG_LEVEL > 0)
     SECTION("std::map-backed object_t")
     {
         using bad_alloc_json = nlohmann::basic_json<std::map,
@@ -401,6 +406,7 @@ TEST_CASE("copy of a deeply nested value survives a failing allocation (#5640)")
         check_deep_copy_survives_failing_allocation<bad_alloc_ordered_json>(false);
         check_deep_copy_survives_failing_allocation<bad_alloc_ordered_json>(true);
     }
+#endif
 }
 
 namespace
@@ -493,9 +499,13 @@ struct countdown_allocator : std::allocator<T>
     template<class U, class... Args>
     void construct(U* p, Args&& ... args)
     {
-        if (constructions_until_failure != 0 && --constructions_until_failure == 0)
+        if (constructions_until_failure != 0)
         {
-            throw std::bad_alloc();
+            --constructions_until_failure;
+            if (constructions_until_failure == 0)
+            {
+                throw std::bad_alloc();
+            }
         }
 
         ::new (static_cast<void*>(p)) U(std::forward<Args>(args)...);

tests/src/unit-alt-string.cpp

@@ -16,6 +16,10 @@
 #include <utility>
 #include <vector>
 
+// NLOHMANN_JSON_SERIALIZE_ENUM_STRICT uses a static std::pair
+DOCTEST_CLANG_SUPPRESS_WARNING_PUSH
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wexit-time-destructors")
+
 /* forward declarations */
 class alt_string;
 bool operator<(const char* op1, const alt_string& op2) noexcept; // NOLINT(misc-use-internal-linkage)
@@ -174,7 +178,7 @@ bool operator<(const char* op1, const alt_string& op2) noexcept
     return op1 < op2.str_impl;
 }
 
-enum class alt_color { red, green };
+enum class alt_color { red, green }; // NOLINT(misc-use-internal-linkage)
 
 // NOLINTNEXTLINE(misc-use-internal-linkage,misc-const-correctness,cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) - false positive
 NLOHMANN_JSON_SERIALIZE_ENUM_STRICT(alt_color,
@@ -434,3 +438,5 @@ TEST_CASE("alternative string type")
         CHECK_THROWS_WITH_AS(_ = doc.get<alt_color>(), "[json.exception.out_of_range.410] enum value out of range for alt_color: \"blue\"", alt_json::out_of_range&);
     }
 }
+
+DOCTEST_CLANG_SUPPRESS_WARNING_POP

tests/src/unit-bjdata.cpp

@@ -67,10 +67,9 @@ TEST_CASE("BJData")
 {
     SECTION("binary_reader BJData lookup tables")
     {
+        // both lookups are static member functions
         std::vector<std::uint8_t> const data;
-        auto ia = nlohmann::detail::input_adapter(data);
-        // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg)
-        nlohmann::detail::binary_reader<json, decltype(ia)> const br{std::move(ia), json::input_format_t::bjdata};
+        using reader_t = nlohmann::detail::binary_reader<json, decltype(nlohmann::detail::input_adapter(data))>;
 
         // the excluded optimized-type markers must match binary_writer's
         // is_bjdata_excluded_type_marker(), which encodes the same 8 markers
@@ -78,13 +77,13 @@ TEST_CASE("BJData")
                 {'[', '{', 'S', 'H', 'T', 'F', 'N', 'Z'
                 })
         {
-            CHECK(br.is_bjd_excluded_optimized_type(marker));
+            CHECK(reader_t::is_bjd_excluded_optimized_type(static_cast<unsigned char>(marker)));
         }
         for (const char marker :
                 {'U', 'i', 'u', 'I', 'm', 'l', 'M', 'L', 'd', 'D', 'C', 'B', 'x'
                 })
         {
-            CHECK(!br.is_bjd_excluded_optimized_type(marker));
+            CHECK(!reader_t::is_bjd_excluded_optimized_type(static_cast<unsigned char>(marker)));
         }
 
         // every dtype marker must round-trip to its ND-array type name
@@ -96,11 +95,11 @@ TEST_CASE("BJData")
         };
         for (const auto& type : types)
         {
-            const char* name = br.bjd_type_name(type.first);
+            const char* name = reader_t::bjd_type_name(static_cast<unsigned char>(type.first));
             REQUIRE(name != nullptr);
             CHECK(std::string(name) == type.second);
         }
-        CHECK(br.bjd_type_name('x') == nullptr);
+        CHECK(reader_t::bjd_type_name(static_cast<unsigned char>('x')) == nullptr);
     }
 
     SECTION("individual values")

tests/src/unit-bon8.cpp

@@ -459,6 +459,47 @@ TEST_CASE("BON8")
             CHECK_THROWS_WITH_AS(_ = json::from_bon8(bytes{0x87, 'a'}), "[json.exception.parse_error.110] parse error at byte 3: syntax error while parsing BON8 string: unexpected end of input", json::parse_error&);
         }
 
+        SECTION("input that ends after a UTF-8 lead byte")
+        {
+            // the lead byte begins either a character or an integer; both are
+            // incomplete, so the lead byte must not end the string before it
+            for (const bool strict :
+                    {
+                        true, false
+                    })
+            {
+                CAPTURE(strict)
+                CHECK_THROWS_WITH_AS(_ = json::from_bon8(bytes{'a', 0xC3}, strict), "[json.exception.parse_error.110] parse error at byte 3: syntax error while parsing BON8 string: unexpected end of input", json::parse_error&);
+                CHECK_THROWS_WITH_AS(_ = json::from_bon8(bytes{0x81, 'a', 0xC3}, strict), "[json.exception.parse_error.110] parse error at byte 4: syntax error while parsing BON8 string: unexpected end of input", json::parse_error&);
+                CHECK_THROWS_WITH_AS(_ = json::from_bon8(bytes{0x81, 'a', 0xE2}, strict), "[json.exception.parse_error.110] parse error at byte 4: syntax error while parsing BON8 string: unexpected end of input", json::parse_error&);
+                CHECK_THROWS_WITH_AS(_ = json::from_bon8(bytes{0x81, 'a', 0xF0}, strict), "[json.exception.parse_error.110] parse error at byte 4: syntax error while parsing BON8 string: unexpected end of input", json::parse_error&);
+                CHECK_THROWS_WITH_AS(_ = json::from_bon8(bytes{0x81, 0xC3, 0xA9, 0xC3}, strict), "[json.exception.parse_error.110] parse error at byte 5: syntax error while parsing BON8 string: unexpected end of input", json::parse_error&);
+                CHECK_THROWS_WITH_AS(_ = json::from_bon8(bytes{0x87, 'a', 0xC3}, strict), "[json.exception.parse_error.110] parse error at byte 4: syntax error while parsing BON8 string: unexpected end of input", json::parse_error&);
+                CHECK_THROWS_WITH_AS(_ = json::from_bon8(bytes{0x87, 0xC3}, strict), "[json.exception.parse_error.110] parse error at byte 3: syntax error while parsing BON8 key: unexpected end of input", json::parse_error&);
+                CHECK_THROWS_WITH_AS(_ = json::from_bon8(bytes{0x88, 'a', 0x91, 0xE2}, strict), "[json.exception.parse_error.110] parse error at byte 5: syntax error while parsing BON8 key: unexpected end of input", json::parse_error&);
+            }
+        }
+
+        SECTION("a message that is cut off is not read as a shorter value")
+        {
+            const json values = {"\xC3\xA9", "a\xE2\x82\xAC", "\xF0\x9F\x98\x80\xC3\xA9", {"a\xC3\xA9"}, {{"\xC3\xA9", "\xE2\x82\xAC"}}, {{"a", {"b\xC3\xA9", 1}}}};
+            for (const auto& j : values)
+            {
+                const bytes message = json::to_bon8(j);
+                for (std::size_t length = 0; length < message.size(); ++length)
+                {
+                    CAPTURE(j)
+                    CAPTURE(length)
+                    bytes prefix = message;
+                    prefix.resize(length);
+                    CHECK(json::from_bon8(prefix, false, false).is_discarded());
+                    // a stream is read byte by byte rather than in bulk
+                    std::istringstream stream(str(prefix));
+                    CHECK(json::from_bon8(stream, false, false).is_discarded());
+                }
+            }
+        }
+
         SECTION("invalid UTF-8")
         {
             // overlong

tests/src/unit-class_parser.cpp

@@ -2001,7 +2001,7 @@ TEST_CASE("parser class")
             const json j = json::parse(R"({"skip": {"k1": 1, "k2": [2, {"k3": 3}]}, "keep": 1})",
                                        [&](int depth, json::parse_event_t event, json & parsed)
             {
-                static const char* const names[] = {"object_start", "object_end", "array_start", "array_end", "key", "value"};
+                static const char* const names[] = {"object_start", "object_end", "array_start", "array_end", "key", "value"}; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays)
                 log.push_back(std::to_string(depth) + " " + names[static_cast<int>(event)] + " " + parsed.dump());
 
                 if (depth == 1 && event == json::parse_event_t::object_start && first)
@@ -2036,7 +2036,7 @@ TEST_CASE("parser class")
             // further effect")
             const auto record = [](std::vector<std::string>& log, int depth, json::parse_event_t event, const json & parsed)
             {
-                static const char* const names[] = {"object_start", "object_end", "array_start", "array_end", "key", "value"};
+                static const char* const names[] = {"object_start", "object_end", "array_start", "array_end", "key", "value"}; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays)
                 log.push_back(std::to_string(depth) + " " + names[static_cast<int>(event)] + " " + parsed.dump());
             };
 

tests/src/unit-constructor2.cpp

@@ -85,7 +85,7 @@ TEST_CASE("other constructors and destructor")
         CHECK(j.type() == json::value_t::object);
         const json k(std::move(j));
         CHECK(k.type() == json::value_t::object);
-        CHECK(j.type() == json::value_t::null); // NOLINT(bugprone-use-after-move,hicpp-invalid-access-moved) access after move is OK here
+        CHECK(j.type() == json::value_t::null); // NOLINT(bugprone-use-after-move,hicpp-invalid-access-moved,clang-analyzer-cplusplus.Move) access after move is OK here
     }
 
     SECTION("copy assignment")

tests/src/unit-conversions.cpp

@@ -1571,9 +1571,15 @@ TEST_CASE("value conversion")
 
                 json const j7 = {0, 1, 2, 3};
                 json const j8 = 2;
+#if JSON_DIAGNOSTICS
+                CHECK_THROWS_WITH_AS((j7.get<std::map<int, int>>()),
+                                     "[json.exception.type_error.302] (/0) type must be array, "
+                                     "but is number", json::type_error&);
+#else
                 CHECK_THROWS_WITH_AS((j7.get<std::map<int, int>>()),
                                      "[json.exception.type_error.302] type must be array, "
                                      "but is number", json::type_error&);
+#endif
                 CHECK_THROWS_WITH_AS((j8.get<std::map<int, int>>()),
                                      "[json.exception.type_error.302] type must be array, "
                                      "but is number", json::type_error&);
@@ -1596,9 +1602,15 @@ TEST_CASE("value conversion")
 
                 json const j7 = {0, 1, 2, 3};
                 json const j8 = 2;
+#if JSON_DIAGNOSTICS
+                CHECK_THROWS_WITH_AS((j7.get<std::unordered_map<int, int>>()),
+                                     "[json.exception.type_error.302] (/0) type must be array, "
+                                     "but is number", json::type_error&);
+#else
                 CHECK_THROWS_WITH_AS((j7.get<std::unordered_map<int, int>>()),
                                      "[json.exception.type_error.302] type must be array, "
                                      "but is number", json::type_error&);
+#endif
                 CHECK_THROWS_WITH_AS((j8.get<std::unordered_map<int, int>>()),
                                      "[json.exception.type_error.302] type must be array, "
                                      "but is number", json::type_error&);
@@ -1747,10 +1759,10 @@ TEST_CASE("Strict JSON to enum mapping")
         CHECK(json(strict_cards::karo) == "karo");
 
         // json -> enum
-        CHECK(strict_cards::kreuz == json("kreuz"));
-        CHECK(strict_cards::pik == json("pik"));
-        CHECK(strict_cards::herz == json("herz"));
-        CHECK(strict_cards::karo == json("karo"));
+        CHECK(json("kreuz").get<strict_cards>() == strict_cards::kreuz);
+        CHECK(json("pik").get<strict_cards>() == strict_cards::pik);
+        CHECK(json("herz").get<strict_cards>() == strict_cards::herz);
+        CHECK(json("karo").get<strict_cards>() == strict_cards::karo);
 
         // invalid json -> exception thrown
         json _;
@@ -1775,10 +1787,10 @@ TEST_CASE("Strict JSON to enum mapping")
         CHECK(json(STRICT_TS_INVALID) == json());
 
         // json -> enum
-        CHECK(STRICT_TS_STOPPED == json("stopped"));
-        CHECK(STRICT_TS_RUNNING == json("running"));
-        CHECK(STRICT_TS_COMPLETED == json("completed"));
-        CHECK(STRICT_TS_INVALID == json());
+        CHECK(json("stopped").get<StrictTaskState>() == STRICT_TS_STOPPED);
+        CHECK(json("running").get<StrictTaskState>() == STRICT_TS_RUNNING);
+        CHECK(json("completed").get<StrictTaskState>() == STRICT_TS_COMPLETED);
+        CHECK(json().get<StrictTaskState>() == STRICT_TS_INVALID);
 
         // invalid json -> exception thrown
         json _;
@@ -1910,6 +1922,8 @@ TEST_CASE("std::optional")
 
         CHECK(json(opt_string) == j_string);
         CHECK(std::optional<std::string>(j_string) == opt_string);
+        // false positive: Infer attributes the destruction of the temporaries above to opt_string
+        // @infer-ignore USE_AFTER_DELETE
     }
 
     SECTION("bool")
@@ -1960,8 +1974,8 @@ TEST_CASE("std::optional")
         CHECK_THROWS_WITH_AS(json(opt), "cannot serialize throwing_to_json_type", std::runtime_error&);
 
         // the conversion is noexcept exactly when converting the contained value is
-        static_assert(!std::is_nothrow_constructible<json, const std::optional<throwing_to_json_type>&>::value, "");
-        static_assert(std::is_nothrow_constructible<json, const std::optional<int>&>::value, "");
+        static_assert(!std::is_nothrow_constructible<json, const std::optional<throwing_to_json_type>&>::value);
+        static_assert(std::is_nothrow_constructible<json, const std::optional<int>&>::value);
     }
 #endif
 }

tests/src/unit-custom-base-class.cpp

@@ -234,7 +234,7 @@ TEST_CASE("JSON Node Metadata")
         // travel with it, just as it does for copy, move, and assignment
         using json = json_with_metadata<int>;
         std::vector<json> values;
-        for (int v :
+        for (const int v :
                 {
                     5, 3, 9, 1, 7, 2, 8, 4, 6, 0, 15, 13, 19, 11, 17, 12, 18, 14, 16, 10,
                     25, 23, 29, 21, 27, 22, 28, 24, 26, 20, 35, 33

tests/src/unit-diagnostics.cpp

@@ -77,6 +77,8 @@ TEST_CASE("Better diagnostics")
     SECTION("Parse error")
     {
         json _;
+        // false positive: a default-constructed json is a valid null value
+        // @infer-ignore NULLPTR_DEREFERENCE
         CHECK_THROWS_WITH_AS(_ = json::parse(""), "[json.exception.parse_error.101] parse error at line 1, column 1: attempting to parse an empty input; check that your input string or stream contains the expected JSON", json::parse_error);
     }
 

tests/src/unit-locale-cpp.cpp

@@ -423,6 +423,7 @@ struct LocaleSwitchingStreambuf final : std::streambuf
     std::string locale_after_first_write;
     bool switched = false;
 
+  protected:
     std::streamsize xsputn(const char* s, std::streamsize n) override
     {
         if (!switched)

tests/src/unit-noexcept.cpp

@@ -60,9 +60,9 @@ TEST_CASE("noexcept")
 {
     // silence -Wunneeded-internal-declaration errors
     static_cast<void>(static_cast<void(*)(json&, pod)>(&to_json));
-    static_cast<void>(static_cast<void(*)(json&, pod_bis)>(&to_json));
+    static_cast<void>(static_cast<void(*)(json&, pod_bis)>(&to_json)); // NOLINT(readability-redundant-casting): selects the overload
     static_cast<void>(static_cast<void(*)(const json&, pod)>(&from_json));
-    static_cast<void>(static_cast<void(*)(const json&, pod_bis)>(&from_json));
+    static_cast<void>(static_cast<void(*)(const json&, pod_bis)>(&from_json)); // NOLINT(readability-redundant-casting): selects the overload
 
     SECTION("nothrow-copy-constructible exceptions")
     {

tests/src/unit-pointer_access.cpp

@@ -79,6 +79,8 @@ TEST_CASE("pointer access")
         // check if pointers are returned correctly
         const test_type* p1 = value.get_ptr<test_type*>();
         CHECK(p1 == value.get_ptr<test_type*>());
+        // false positive: p1 is non-null, as value has type test_type
+        // @infer-ignore NULLPTR_DEREFERENCE
         CHECK(*p1 == value.get<test_type>());
 
         const test_type* p2 = value.get_ptr<const test_type*>();
@@ -108,6 +110,8 @@ TEST_CASE("pointer access")
         // check if pointers are returned correctly
         test_type* p1 = value.get_ptr<test_type*>();
         CHECK(p1 == value.get_ptr<test_type*>());
+        // false positive: p1 is non-null, as value has type test_type
+        // @infer-ignore NULLPTR_DEREFERENCE
         CHECK(*p1 == value.get<test_type>());
 
         const test_type* p2 = value.get_ptr<const test_type*>();

tests/src/unit-udt.cpp

@@ -446,6 +446,8 @@ TEST_CASE("adl_serializer specialization" * doctest::test_suite("udt"))
 
             auto optPerson = j.get<std::shared_ptr<udt::person>>();
             REQUIRE(optPerson);
+            // false positive: REQUIRE above guarantees optPerson is non-null
+            // @infer-ignore NULLPTR_DEREFERENCE
             CHECK(*optPerson == person);
 
             j = nullptr;
@@ -559,6 +561,8 @@ TEST_CASE("Non-copyable types" * doctest::test_suite("udt"))
 
         auto optPerson = j.get<std::unique_ptr<udt::person>>();
         REQUIRE(optPerson);
+        // false positive: REQUIRE above guarantees optPerson is non-null
+        // @infer-ignore NULLPTR_DEREFERENCE
         CHECK(*optPerson == person);
 
         j = nullptr;