Merge develop into fix/issue-5993

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

include/nlohmann/detail/output/serializer.hpp

@@ -8,7 +8,7 @@
 
 #pragma once
 
-#include <algorithm> // remove, fill, find, none_of, min
+#include <algorithm> // copy, none_of, min, search
 #include <array> // array
 #include <clocale> // localeconv, lconv
 #include <cmath> // isfinite
@@ -2080,31 +2080,35 @@ class serializer
         // check if the buffer was large enough
         JSON_ASSERT(static_cast<std::size_t>(len) < number_buffer.size());
 
-        // look up the locale's thousands separator and decimal point now,
+        // look up the locale's decimal point now, matching what
-        // matching what snprintf_float() just used (see lexer::get_decimal_point())
+        // snprintf_float() just used (see get_decimal_point() in
+        // number_parse.hpp); it is a string, which can be longer than one
+        // byte (U+066B in ps_AF, #5993).
+        // A thousands separator needs no handling: %g never groups digits.
         const auto* loc = std::localeconv();
         JSON_ASSERT(loc != nullptr);
-        const char thousands_sep = (loc->thousands_sep == nullptr) ? '\0' : *loc->thousands_sep;
+        const char* const decimal_point = (loc->decimal_point == nullptr) ? "" : loc->decimal_point;
-        const char decimal_point = (loc->decimal_point == nullptr) ? '\0' : *loc->decimal_point;
+        const std::size_t decimal_point_len = std::char_traits<char>::length(decimal_point);
-
-        // erase thousands separators
-        if (thousands_sep != '\0')
-        {
-            // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::remove returns an iterator, see https://github.com/nlohmann/json/issues/3081
-            const auto end = std::remove(number_buffer.begin(), number_buffer.begin() + len, thousands_sep);
-            std::fill(end, number_buffer.end(), '\0');
-            JSON_ASSERT((end - number_buffer.begin()) <= len);
-            len = (end - number_buffer.begin());
-        }
 
         // convert decimal point to '.'
-        if (decimal_point != '\0' && decimal_point != '.')
+        if ((decimal_point_len > 1) || (decimal_point_len == 1 && decimal_point[0] != '.'))
         {
-            // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::find returns an iterator, see https://github.com/nlohmann/json/issues/3081
+            const auto first = number_buffer.begin();
-            const auto dec_pos = std::find(number_buffer.begin(), number_buffer.end(), decimal_point);
+            const auto last = first + len;
-            if (dec_pos != number_buffer.end())
+            // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::search returns an iterator, see https://github.com/nlohmann/json/issues/3081
+            const auto dec_pos = std::search(first, last, decimal_point, decimal_point + decimal_point_len);
+            if (dec_pos != last)
             {
                 *dec_pos = '.';
+                if (decimal_point_len > 1)
+                {
+                    // move the fraction and exponent behind the '.' (std::copy
+                    // allows the overlap, as the destination is before the source)
+                    // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::copy returns an iterator, see https://github.com/nlohmann/json/issues/3081
+                    const auto end = std::copy(dec_pos + static_cast<std::ptrdiff_t>(decimal_point_len), last, dec_pos + 1);
+                    *end = '\0';
+                    len = end - first;
+                }
             }
         }
 

single_include/nlohmann/json.hpp

@@ -33446,7 +33446,7 @@ NLOHMANN_JSON_NAMESPACE_END
 
 
 
-#include <algorithm> // remove, fill, find, none_of, min
+#include <algorithm> // copy, none_of, min, search
 #include <array> // array
 #include <clocale> // localeconv, lconv
 #include <cmath> // isfinite
@@ -37300,31 +37300,35 @@ class serializer
         // check if the buffer was large enough
         JSON_ASSERT(static_cast<std::size_t>(len) < number_buffer.size());
 
-        // look up the locale's thousands separator and decimal point now,
+        // look up the locale's decimal point now, matching what
-        // matching what snprintf_float() just used (see lexer::get_decimal_point())
+        // snprintf_float() just used (see get_decimal_point() in
+        // number_parse.hpp); it is a string, which can be longer than one
+        // byte (U+066B in ps_AF, #5993).
+        // A thousands separator needs no handling: %g never groups digits.
         const auto* loc = std::localeconv();
         JSON_ASSERT(loc != nullptr);
-        const char thousands_sep = (loc->thousands_sep == nullptr) ? '\0' : *loc->thousands_sep;
+        const char* const decimal_point = (loc->decimal_point == nullptr) ? "" : loc->decimal_point;
-        const char decimal_point = (loc->decimal_point == nullptr) ? '\0' : *loc->decimal_point;
+        const std::size_t decimal_point_len = std::char_traits<char>::length(decimal_point);
-
-        // erase thousands separators
-        if (thousands_sep != '\0')
-        {
-            // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::remove returns an iterator, see https://github.com/nlohmann/json/issues/3081
-            const auto end = std::remove(number_buffer.begin(), number_buffer.begin() + len, thousands_sep);
-            std::fill(end, number_buffer.end(), '\0');
-            JSON_ASSERT((end - number_buffer.begin()) <= len);
-            len = (end - number_buffer.begin());
-        }
 
         // convert decimal point to '.'
-        if (decimal_point != '\0' && decimal_point != '.')
+        if ((decimal_point_len > 1) || (decimal_point_len == 1 && decimal_point[0] != '.'))
         {
-            // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::find returns an iterator, see https://github.com/nlohmann/json/issues/3081
+            const auto first = number_buffer.begin();
-            const auto dec_pos = std::find(number_buffer.begin(), number_buffer.end(), decimal_point);
+            const auto last = first + len;
-            if (dec_pos != number_buffer.end())
+            // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::search returns an iterator, see https://github.com/nlohmann/json/issues/3081
+            const auto dec_pos = std::search(first, last, decimal_point, decimal_point + decimal_point_len);
+            if (dec_pos != last)
             {
                 *dec_pos = '.';
+                if (decimal_point_len > 1)
+                {
+                    // move the fraction and exponent behind the '.' (std::copy
+                    // allows the overlap, as the destination is before the source)
+                    // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::copy returns an iterator, see https://github.com/nlohmann/json/issues/3081
+                    const auto end = std::copy(dec_pos + static_cast<std::ptrdiff_t>(decimal_point_len), last, dec_pos + 1);
+                    *end = '\0';
+                    len = end - first;
+                }
             }
         }
 

tests/src/unit-allocator.cpp

@@ -1038,14 +1038,6 @@ template<class T>
 struct sweep_allocator
 {
     using value_type = T;
-    // GCC 4.8's std::map takes these from the allocator instead of from
-    // std::allocator_traits
-    using pointer = T*;
-    using const_pointer = const T*;
-    using reference = T&;
-    using const_reference = const T&;
-    using size_type = std::size_t;
-    using difference_type = std::ptrdiff_t;
 
     sweep_allocator() = default;
     template<class U> sweep_allocator(const sweep_allocator<U>& /*unused*/) noexcept {}
@@ -1163,9 +1155,8 @@ TEST_CASE("parse() survives a failing allocation at every point (#5985)")
 
     // With iterator debugging (MSVC STL debug builds, also used by clang-cl),
     // containers allocate a debug proxy through the allocator inside their
-    // noexcept default and move constructors, so failing that allocation
+    // noexcept move constructors, so failing that allocation terminates the
-    // terminates the program instead of throwing std::bad_alloc. Nothing to
+    // program instead of throwing std::bad_alloc. Nothing to check there.
-    // check there.
 #if !(defined(_ITERATOR_DEBUG_LEVEL) && _ITERATOR_DEBUG_LEVEL > 0)
     SECTION("parser callback")
     {
@@ -1187,6 +1178,7 @@ TEST_CASE("parse() survives a failing allocation at every point (#5985)")
             return sweep_json::parse(text, drop_d);
         });
     }
+#endif
 
     SECTION("sax_parse into an existing value")
     {
@@ -1200,6 +1192,5 @@ TEST_CASE("parse() survives a failing allocation at every point (#5985)")
             return value;
         });
     }
-#endif
 }
 #endif

tests/src/unit-class_lexer.cpp

@@ -2222,10 +2222,9 @@ TEST_CASE("serialization of short strings")
     {
         CAPTURE(s)
         const json j = s;
-        const std::string quoted = "\"" + s + "\""; // NOLINT(performance-inefficient-string-concatenation)
+        CHECK(j.dump() == "\"" + s + "\"");
-        CHECK(j.dump() == quoted);
+        CHECK(j.dump(-1, ' ', true) == "\"" + s + "\"");
-        CHECK(j.dump(-1, ' ', true) == quoted);
+        CHECK(json({{s, s}}).dump() == "{\"" + s + "\":\"" + s + "\"}");
-        CHECK(json({{s, s}}).dump() == "{" + quoted + ":" + quoted + "}"); // NOLINT(performance-inefficient-string-concatenation)
     }
 
     // a character that needs an escape

tests/src/unit-json_view.cpp

@@ -111,7 +111,7 @@ struct failing_allocator
     template<typename U>
     explicit failing_allocator(const failing_allocator<U>& /*other*/) noexcept {}
 
-    [[noreturn]] T* allocate(std::size_t /*n*/) // NOLINT(readability-convert-member-functions-to-static)
+    T* allocate(std::size_t /*n*/) // NOLINT(readability-convert-member-functions-to-static)
     {
         throw std::bad_alloc();
     }

tests/src/unit-locale-cpp.cpp

@@ -397,6 +397,14 @@ TEST_CASE("locale with a multi-byte decimal point")
 
         // a value the locale-independent paths convert is not affected
         CHECK(json::parse("12.5") == 12.5);
+
+        // dump() of a long double that takes the snprintf path replaces the
+        // whole decimal point, not only its first byte (#5993)
+        CHECK(long_double_json(1.5L).dump() == "1.5");
+        CHECK(long_double_json(2.0L).dump() == "2.0");
+        CHECK(long_double_json(-0.25L).dump() == "-0.25");
+        CHECK(long_double_json::parse(long_double_json(1.5e-7L).dump()) == long_double_json(1.5e-7L));
+        CHECK(long_double_json::parse(ld.dump()) == expected_long_double);
     }
     if (!tested)
     {

tests/src/unit-msgpack-cpp17.cpp

@@ -122,8 +122,7 @@ TEST_CASE("JSON text with std::byte")
     SECTION("invalid text")
     {
         CHECK_FALSE(json::accept(to_bytes("[1,\n2,")));
-        json _;
+        CHECK_THROWS_WITH_AS(json::parse(to_bytes("[1,\n2,")),
-        CHECK_THROWS_WITH_AS(_ = json::parse(to_bytes("[1,\n2,")),
                              "[json.exception.parse_error.101] parse error at line 2, column 3: syntax error while parsing value - unexpected end of input; expected '[', '{', or a literal", json::parse_error&);
     }
 }