Compare commits

..
Author SHA1 Message Date
Niels Lohmann 0395ad0992 Pass the complete text of a UBJSON/BJData high-precision number to SAX
For a high-precision number that is a floating-point number, the
UBJSON/BJData reader passed lexer::get_token_string() as the source
text to SAX number_float(). That helper is meant for error messages and
shows a token of more than 8192 characters only by its first and last
4096 characters around "...", so a SAX handler received a shortened
string for a long number.

Pass the number text the lexer collected while scanning the number
(lexer::get_string()) instead. It is complete and, as before, holds
only the number, without any whitespace or byte order mark around it
in the payload. Error messages still use get_token_string().

Signed-off-by: Niels Lohmann <mail@nlohmann.me>
2026-11-16 08:27:07 +01:00
10 changed files with 64 additions and 28 deletions

No files matched your search

@@ -48,11 +48,10 @@ namespace detail
{
/*!
@brief implements binary to decimal floating-point conversion: the shortest
digits of Zmij (see zmij.hpp) for IEEE 754 binary64 numbers, and the Grisu2
algorithm for all other formats.
@brief implements the Grisu2 algorithm for binary to decimal floating-point
conversion.
The Grisu2 implementation is a slightly modified version of the reference
This implementation is a slightly modified version of the reference
implementation which may be obtained from
http://florian.loitsch.com/publications (bench.tar.gz).
@@ -4096,6 +4096,10 @@ class binary_reader
auto number_lexer = detail::lexer<BasicJsonType, ia_type>(detail::input_adapter(number_vector), false);
const auto result_number = number_lexer.scan();
const auto number_string = number_lexer.get_token_string();
// the number as the lexer read it, for SAX number_float(): unlike
// get_token_string(), which is meant for error messages and shortens
// long tokens, it is complete; the next scan() replaces it
const string_t number_text = number_lexer.get_string();
const auto result_remainder = number_lexer.scan();
using token_type = typename detail::lexer_base<BasicJsonType>::token_type;
@@ -4122,10 +4126,7 @@ class binary_reader
number_string,
out_of_range::create(exception_id::number_overflow, concat("number overflow parsing '", number_string, '\''), nullptr));
}
// number_string is a std::string, while the SAX interface takes a
// string_t; convert explicitly, as the two are only implicitly
// convertible for some string types
return sax->number_float(parsed_float, string_t(number_string.data(), number_string.size()));
return sax->number_float(parsed_float, number_text);
}
case token_type::uninitialized:
case token_type::literal_true:
@@ -257,7 +257,7 @@ struct ieee_binary_format<53> // binary64
These formats (float, double, and long double where it is binary64, e.g.
with MSVC or on Apple arm64) are converted by parse_float_native(). The
predicate is the one the serializer uses to choose to_chars() over snprintf.
predicate is the one the serializer uses to choose Grisu2.
*/
template<typename FloatType>
struct has_native_float_format
@@ -915,7 +915,7 @@ class binary_writer
JSON_THROW(out_of_range::create(exception_id::subtype_out_of_range, concat("subtype ", std::to_string(subtype), " is too large for ", what, " (max 255)"), &j));
}
/// @throw other_error.502 always
/// @throw other_error.407 always
JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE
static void throw_size_marker_required(const BasicJsonType& j)
{
@@ -2033,9 +2033,8 @@ class serializer
}
// If number_float_t is an IEEE-754 single or double precision number,
// use to_chars() to produce short numbers which are guaranteed to
// round-trip, using strtof and strtod, resp. (the shortest digits of
// Zmij for binary64, the Grisu2 algorithm for binary32).
// use the Grisu2 algorithm to produce short numbers which are
// guaranteed to round-trip, using strtof and strtod, resp.
//
// NB: The test below works if <long double> == <double>.
static constexpr bool is_ieee_single_or_double
+1 -1
View File
@@ -849,7 +849,7 @@ class basic_json_view
return !navigation::editable() || m_doc->edits == nullptr;
}
/// the member of this object with @a key (the first one if the key occurs more than once), or nullptr
/// the member of this object with @a key (the last one if the key occurs more than once), or nullptr
NLOHMANN_VIEW_ALWAYS_INLINE const node* find_key(string_view_t key) const
{
return find_key(key, std::integral_constant<bool, is_editable> {});
+12 -13
View File
@@ -11642,7 +11642,7 @@ struct ieee_binary_format<53> // binary64
These formats (float, double, and long double where it is binary64, e.g.
with MSVC or on Apple arm64) are converted by parse_float_native(). The
predicate is the one the serializer uses to choose to_chars() over snprintf.
predicate is the one the serializer uses to choose Grisu2.
*/
template<typename FloatType>
struct has_native_float_format
@@ -22774,6 +22774,10 @@ class binary_reader
auto number_lexer = detail::lexer<BasicJsonType, ia_type>(detail::input_adapter(number_vector), false);
const auto result_number = number_lexer.scan();
const auto number_string = number_lexer.get_token_string();
// the number as the lexer read it, for SAX number_float(): unlike
// get_token_string(), which is meant for error messages and shortens
// long tokens, it is complete; the next scan() replaces it
const string_t number_text = number_lexer.get_string();
const auto result_remainder = number_lexer.scan();
using token_type = typename detail::lexer_base<BasicJsonType>::token_type;
@@ -22800,10 +22804,7 @@ class binary_reader
number_string,
out_of_range::create(exception_id::number_overflow, concat("number overflow parsing '", number_string, '\''), nullptr));
}
// number_string is a std::string, while the SAX interface takes a
// string_t; convert explicitly, as the two are only implicitly
// convertible for some string types
return sax->number_float(parsed_float, string_t(number_string.data(), number_string.size()));
return sax->number_float(parsed_float, number_text);
}
case token_type::uninitialized:
case token_type::literal_true:
@@ -31147,7 +31148,7 @@ class binary_writer
JSON_THROW(out_of_range::create(exception_id::subtype_out_of_range, concat("subtype ", std::to_string(subtype), " is too large for ", what, " (max 255)"), &j));
}
/// @throw other_error.502 always
/// @throw other_error.407 always
JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE
static void throw_size_marker_required(const BasicJsonType& j)
{
@@ -36006,11 +36007,10 @@ namespace detail
{
/*!
@brief implements binary to decimal floating-point conversion: the shortest
digits of Zmij (see zmij.hpp) for IEEE 754 binary64 numbers, and the Grisu2
algorithm for all other formats.
@brief implements the Grisu2 algorithm for binary to decimal floating-point
conversion.
The Grisu2 implementation is a slightly modified version of the reference
This implementation is a slightly modified version of the reference
implementation which may be obtained from
http://florian.loitsch.com/publications (bench.tar.gz).
@@ -39520,9 +39520,8 @@ class serializer
}
// If number_float_t is an IEEE-754 single or double precision number,
// use to_chars() to produce short numbers which are guaranteed to
// round-trip, using strtof and strtod, resp. (the shortest digits of
// Zmij for binary64, the Grisu2 algorithm for binary32).
// use the Grisu2 algorithm to produce short numbers which are
// guaranteed to round-trip, using strtof and strtod, resp.
//
// NB: The test below works if <long double> == <double>.
static constexpr bool is_ieee_single_or_double
+1 -1
View File
@@ -11016,7 +11016,7 @@ class basic_json_view
return !navigation::editable() || m_doc->edits == nullptr;
}
/// the member of this object with @a key (the first one if the key occurs more than once), or nullptr
/// the member of this object with @a key (the last one if the key occurs more than once), or nullptr
NLOHMANN_VIEW_ALWAYS_INLINE const node* find_key(string_view_t key) const
{
return find_key(key, std::integral_constant<bool, is_editable> {});
+19
View File
@@ -1278,6 +1278,25 @@ TEST_CASE("BJData")
CHECK(j.dump() == "3.141592653589793");
}
SECTION("long floating-point number")
{
// the SAX parser receives the number text as it is in the
// input, also when it is longer than the part of a token
// that error messages show
const std::vector<std::size_t> lengths = {8192, 8193, 10000};
for (const auto length : lengths)
{
CAPTURE(length)
const std::string number = "1." + std::string(length - 2, '3');
std::vector<std::uint8_t> vec = {'H', 'I', static_cast<std::uint8_t>(length & 0xFF), static_cast<std::uint8_t>(length >> 8)};
vec.insert(vec.end(), number.begin(), number.end());
SaxEventLogger logger;
CHECK(json::sax_parse(vec, &logger, json::input_format_t::bjdata));
CHECK(logger.events == std::vector<std::string>({"number_float(" + number + ")"}));
CHECK(json::from_bjdata(vec).is_number_float());
}
}
SECTION("errors")
{
// error while parsing length
+19
View File
@@ -793,6 +793,25 @@ TEST_CASE("UBJSON")
CHECK(j.dump() == "3.141592653589793");
}
SECTION("long floating-point number")
{
// the SAX parser receives the number text as it is in the
// input, also when it is longer than the part of a token
// that error messages show
const std::vector<std::size_t> lengths = {8192, 8193, 10000};
for (const auto length : lengths)
{
CAPTURE(length)
const std::string number = "1." + std::string(length - 2, '3');
std::vector<std::uint8_t> vec = {'H', 'I', static_cast<std::uint8_t>(length >> 8), static_cast<std::uint8_t>(length & 0xFF)};
vec.insert(vec.end(), number.begin(), number.end());
SaxEventLogger logger;
CHECK(json::sax_parse(vec, &logger, json::input_format_t::ubjson));
CHECK(logger.events == std::vector<std::string>({"number_float(" + number + ")"}));
CHECK(json::from_ubjson(vec).is_number_float());
}
}
SECTION("errors")
{
// error while parsing length