Merge develop into fix/issue-5993

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

include/nlohmann/detail/output/serializer.hpp

@@ -8,7 +8,7 @@
 
 #pragma once
 
-#include <algorithm> // remove, fill, find, none_of, min
+#include <algorithm> // copy, none_of, min, search
 #include <array> // array
 #include <clocale> // localeconv, lconv
 #include <cmath> // isfinite
@@ -2080,31 +2080,35 @@ class serializer
         // check if the buffer was large enough
         JSON_ASSERT(static_cast<std::size_t>(len) < number_buffer.size());
 
-        // look up the locale's thousands separator and decimal point now,
-        // matching what snprintf_float() just used (see lexer::get_decimal_point())
+        // look up the locale's decimal point now, matching what
+        // snprintf_float() just used (see get_decimal_point() in
+        // number_parse.hpp); it is a string, which can be longer than one
+        // byte (U+066B in ps_AF, #5993).
+        // A thousands separator needs no handling: %g never groups digits.
         const auto* loc = std::localeconv();
         JSON_ASSERT(loc != nullptr);
-        const char thousands_sep = (loc->thousands_sep == nullptr) ? '\0' : *loc->thousands_sep;
-        const char decimal_point = (loc->decimal_point == nullptr) ? '\0' : *loc->decimal_point;
-
-        // erase thousands separators
-        if (thousands_sep != '\0')
-        {
-            // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::remove returns an iterator, see https://github.com/nlohmann/json/issues/3081
-            const auto end = std::remove(number_buffer.begin(), number_buffer.begin() + len, thousands_sep);
-            std::fill(end, number_buffer.end(), '\0');
-            JSON_ASSERT((end - number_buffer.begin()) <= len);
-            len = (end - number_buffer.begin());
-        }
+        const char* const decimal_point = (loc->decimal_point == nullptr) ? "" : loc->decimal_point;
+        const std::size_t decimal_point_len = std::char_traits<char>::length(decimal_point);
 
         // convert decimal point to '.'
-        if (decimal_point != '\0' && decimal_point != '.')
+        if ((decimal_point_len > 1) || (decimal_point_len == 1 && decimal_point[0] != '.'))
         {
-            // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::find returns an iterator, see https://github.com/nlohmann/json/issues/3081
-            const auto dec_pos = std::find(number_buffer.begin(), number_buffer.end(), decimal_point);
-            if (dec_pos != number_buffer.end())
+            const auto first = number_buffer.begin();
+            const auto last = first + len;
+            // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::search returns an iterator, see https://github.com/nlohmann/json/issues/3081
+            const auto dec_pos = std::search(first, last, decimal_point, decimal_point + decimal_point_len);
+            if (dec_pos != last)
             {
                 *dec_pos = '.';
+                if (decimal_point_len > 1)
+                {
+                    // move the fraction and exponent behind the '.' (std::copy
+                    // allows the overlap, as the destination is before the source)
+                    // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::copy returns an iterator, see https://github.com/nlohmann/json/issues/3081
+                    const auto end = std::copy(dec_pos + static_cast<std::ptrdiff_t>(decimal_point_len), last, dec_pos + 1);
+                    *end = '\0';
+                    len = end - first;
+                }
             }
         }
 

include/nlohmann/detail/view/builder.hpp

@@ -1168,20 +1168,20 @@ indent_done:
             if (negative)
             {
                 ++p;
+                if (NLOHMANN_VIEW_UNLIKELY(cur() < '0' || cur() > '9'))
+                {
+                    return fail(error_code::number_after_minus);
+                }
             }
             const unsigned char* const int_start = p;
             if (cur() == '0')
             {
                 ++p;
             }
-            else if (!negative || NLOHMANN_VIEW_LIKELY(cur() >= '1' && cur() <= '9'))
-            {
-                // (without a sign, the dispatch saw a digit)
-                p = skip_digits(p + 1, e);
-            }
             else
             {
-                return fail(error_code::number_after_minus);
+                // (a digit: the dispatch or the check after the sign saw it)
+                p = skip_digits(p + 1, e);
             }
             const auto int_digits = static_cast<std::size_t>(p - int_start);
             std::size_t frac_digits = 0;

include/nlohmann/detail/view/errors.hpp

@@ -251,8 +251,12 @@ std::string diagnostics(const error_site site)
     std::string context;
 #if JSON_DIAGNOSTICS
     std::vector<std::string> tokens;
-    // a value that is not found leaves no tokens, like the root
-    static_cast<void>(edited ? value_path<true>(d, site.value(), tokens) : value_path<false>(d, site.value(), tokens));
+    const bool found = edited ? value_path<true>(d, site.value(), tokens) : value_path<false>(d, site.value(), tokens);
+    if (!found)
+    {
+        // a value that is not found has no path, and with it no bytes
+        return context;
+    }
     if (!tokens.empty())
     {
         // the root value has no path, but still has a byte range

single_include/nlohmann/json.hpp

@@ -33446,7 +33446,7 @@ NLOHMANN_JSON_NAMESPACE_END
 
 
 
-#include <algorithm> // remove, fill, find, none_of, min
+#include <algorithm> // copy, none_of, min, search
 #include <array> // array
 #include <clocale> // localeconv, lconv
 #include <cmath> // isfinite
@@ -37300,31 +37300,35 @@ class serializer
         // check if the buffer was large enough
         JSON_ASSERT(static_cast<std::size_t>(len) < number_buffer.size());
 
-        // look up the locale's thousands separator and decimal point now,
-        // matching what snprintf_float() just used (see lexer::get_decimal_point())
+        // look up the locale's decimal point now, matching what
+        // snprintf_float() just used (see get_decimal_point() in
+        // number_parse.hpp); it is a string, which can be longer than one
+        // byte (U+066B in ps_AF, #5993).
+        // A thousands separator needs no handling: %g never groups digits.
         const auto* loc = std::localeconv();
         JSON_ASSERT(loc != nullptr);
-        const char thousands_sep = (loc->thousands_sep == nullptr) ? '\0' : *loc->thousands_sep;
-        const char decimal_point = (loc->decimal_point == nullptr) ? '\0' : *loc->decimal_point;
-
-        // erase thousands separators
-        if (thousands_sep != '\0')
-        {
-            // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::remove returns an iterator, see https://github.com/nlohmann/json/issues/3081
-            const auto end = std::remove(number_buffer.begin(), number_buffer.begin() + len, thousands_sep);
-            std::fill(end, number_buffer.end(), '\0');
-            JSON_ASSERT((end - number_buffer.begin()) <= len);
-            len = (end - number_buffer.begin());
-        }
+        const char* const decimal_point = (loc->decimal_point == nullptr) ? "" : loc->decimal_point;
+        const std::size_t decimal_point_len = std::char_traits<char>::length(decimal_point);
 
         // convert decimal point to '.'
-        if (decimal_point != '\0' && decimal_point != '.')
+        if ((decimal_point_len > 1) || (decimal_point_len == 1 && decimal_point[0] != '.'))
         {
-            // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::find returns an iterator, see https://github.com/nlohmann/json/issues/3081
-            const auto dec_pos = std::find(number_buffer.begin(), number_buffer.end(), decimal_point);
-            if (dec_pos != number_buffer.end())
+            const auto first = number_buffer.begin();
+            const auto last = first + len;
+            // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::search returns an iterator, see https://github.com/nlohmann/json/issues/3081
+            const auto dec_pos = std::search(first, last, decimal_point, decimal_point + decimal_point_len);
+            if (dec_pos != last)
             {
                 *dec_pos = '.';
+                if (decimal_point_len > 1)
+                {
+                    // move the fraction and exponent behind the '.' (std::copy
+                    // allows the overlap, as the destination is before the source)
+                    // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::copy returns an iterator, see https://github.com/nlohmann/json/issues/3081
+                    const auto end = std::copy(dec_pos + static_cast<std::ptrdiff_t>(decimal_point_len), last, dec_pos + 1);
+                    *end = '\0';
+                    len = end - first;
+                }
             }
         }
 

single_include/nlohmann/json_view.hpp

@@ -2473,20 +2473,20 @@ indent_done:
             if (negative)
             {
                 ++p;
+                if (NLOHMANN_VIEW_UNLIKELY(cur() < '0' || cur() > '9'))
+                {
+                    return fail(error_code::number_after_minus);
+                }
             }
             const unsigned char* const int_start = p;
             if (cur() == '0')
             {
                 ++p;
             }
-            else if (!negative || NLOHMANN_VIEW_LIKELY(cur() >= '1' && cur() <= '9'))
-            {
-                // (without a sign, the dispatch saw a digit)
-                p = skip_digits(p + 1, e);
-            }
             else
             {
-                return fail(error_code::number_after_minus);
+                // (a digit: the dispatch or the check after the sign saw it)
+                p = skip_digits(p + 1, e);
             }
             const auto int_digits = static_cast<std::size_t>(p - int_start);
             std::size_t frac_digits = 0;
@@ -5080,8 +5080,12 @@ std::string diagnostics(const error_site site)
     std::string context;
 #if JSON_DIAGNOSTICS
     std::vector<std::string> tokens;
-    // a value that is not found leaves no tokens, like the root
-    static_cast<void>(edited ? value_path<true>(d, site.value(), tokens) : value_path<false>(d, site.value(), tokens));
+    const bool found = edited ? value_path<true>(d, site.value(), tokens) : value_path<false>(d, site.value(), tokens);
+    if (!found)
+    {
+        // a value that is not found has no path, and with it no bytes
+        return context;
+    }
     if (!tokens.empty())
     {
         // the root value has no path, but still has a byte range

tests/src/unit-locale-cpp.cpp

@@ -397,6 +397,14 @@ TEST_CASE("locale with a multi-byte decimal point")
 
         // a value the locale-independent paths convert is not affected
         CHECK(json::parse("12.5") == 12.5);
+
+        // dump() of a long double that takes the snprintf path replaces the
+        // whole decimal point, not only its first byte (#5993)
+        CHECK(long_double_json(1.5L).dump() == "1.5");
+        CHECK(long_double_json(2.0L).dump() == "2.0");
+        CHECK(long_double_json(-0.25L).dump() == "-0.25");
+        CHECK(long_double_json::parse(long_double_json(1.5e-7L).dump()) == long_double_json(1.5e-7L));
+        CHECK(long_double_json::parse(ld.dump()) == expected_long_double);
     }
     if (!tested)
     {
@@ -505,8 +513,8 @@ struct locale_switching_token
 {
     using value_type = char;
 
-    locale_switching_token(std::string text_, const char* switch_to_)
-        : text(std::move(text_)), switch_to(switch_to_)
+    locale_switching_token(std::string text_, std::string switch_to_)
+        : text(std::move(text_)), switch_to(std::move(switch_to_))
     {}
 
     std::size_t size() const
@@ -523,13 +531,13 @@ struct locale_switching_token
     {
         if (!switched)
         {
-            switched = std::setlocale(LC_NUMERIC, switch_to) != nullptr;
+            switched = std::setlocale(LC_NUMERIC, switch_to.c_str()) != nullptr;
         }
         return &text[0];
     }
 
     std::string text;
-    const char* switch_to;
+    std::string switch_to;
     bool switched = false;
 };