Compare commits

..
Author SHA1 Message Date
Niels Lohmann 0395ad0992 Pass the complete text of a UBJSON/BJData high-precision number to SAX
For a high-precision number that is a floating-point number, the
UBJSON/BJData reader passed lexer::get_token_string() as the source
text to SAX number_float(). That helper is meant for error messages and
shows a token of more than 8192 characters only by its first and last
4096 characters around "...", so a SAX handler received a shortened
string for a long number.

Pass the number text the lexer collected while scanning the number
(lexer::get_string()) instead. It is complete and, as before, holds
only the number, without any whitespace or byte order mark around it
in the payload. Error messages still use get_token_string().

Signed-off-by: Niels Lohmann <mail@nlohmann.me>
2026-11-16 08:27:07 +01:00
5 changed files with 134 additions and 275 deletions

No files matched your search

+48 -76
View File
@@ -1927,21 +1927,6 @@ class binary_reader
return result;
}
/*!
@brief move past a value of @a size bytes, starting with @a head, that a
bulk run reads, as get() and get_number() would
Called before the value is passed on, so that an error reported while
passing it on (see @ref emit_float) has the position and the last byte
that reading the value byte by byte gives.
*/
void bulk_advance(const std::size_t size, const unsigned char head)
{
ia.bulk_skip(size);
chars_read += size;
current = head;
}
/*!
@brief read the integers, floats, booleans, and nulls that follow in a
definite-length array, without the per-element bookkeeping of
@@ -1967,70 +1952,71 @@ class binary_reader
{
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
const unsigned char head = data[0];
std::size_t size = 0;
bool accepted = false;
switch (head)
{
case 0x18:
bulk_advance(2, head);
size = 2;
accepted = emit_unsigned(data[1]);
break;
case 0x19:
bulk_advance(3, head);
size = 3;
accepted = emit_unsigned(load_big_endian<std::uint16_t>(data + 1));
break;
case 0x1A:
bulk_advance(5, head);
size = 5;
accepted = emit_unsigned(load_big_endian<std::uint32_t>(data + 1));
break;
case 0x1B:
bulk_advance(9, head);
size = 9;
accepted = emit_unsigned(load_big_endian<std::uint64_t>(data + 1));
break;
case 0x38:
bulk_advance(2, head);
size = 2;
accepted = emit_cbor_negative_integer(data[1]);
break;
case 0x39:
bulk_advance(3, head);
size = 3;
accepted = emit_cbor_negative_integer(load_big_endian<std::uint16_t>(data + 1));
break;
case 0x3A:
bulk_advance(5, head);
size = 5;
accepted = emit_cbor_negative_integer(load_big_endian<std::uint32_t>(data + 1));
break;
case 0x3B:
bulk_advance(9, head);
size = 9;
accepted = emit_cbor_negative_integer(load_big_endian<std::uint64_t>(data + 1));
break;
case 0xF4:
bulk_advance(1, head);
size = 1;
accepted = sax->boolean(false);
break;
case 0xF5:
bulk_advance(1, head);
size = 1;
accepted = sax->boolean(true);
break;
case 0xF6:
bulk_advance(1, head);
size = 1;
accepted = sax->null();
break;
case 0xFA:
bulk_advance(5, head);
size = 5;
accepted = emit_float(load_big_endian<float>(data + 1));
break;
case 0xFB:
bulk_advance(9, head);
size = 9;
accepted = emit_float(load_big_endian<double>(data + 1));
break;
default:
if (head <= 0x17)
{
bulk_advance(1, head);
size = 1;
accepted = sax->number_unsigned(static_cast<number_unsigned_t>(head));
}
else if (head >= 0x20 && head <= 0x37)
{
bulk_advance(1, head);
size = 1;
accepted = sax->number_integer(static_cast<std::int8_t>(0x20 - 1 - head));
}
else
@@ -2040,6 +2026,10 @@ class binary_reader
}
break;
}
// as if read by get() and get_number(): current is the head byte
ia.bulk_skip(size);
chars_read += size;
current = head;
--remaining;
if (JSON_HEDLEY_UNLIKELY(!accepted))
{
@@ -2877,72 +2867,73 @@ class binary_reader
{
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
const unsigned char head = data[0];
std::size_t size = 0;
bool accepted = false;
switch (head)
{
case 0xC0: // nil
bulk_advance(1, head);
size = 1;
accepted = sax->null();
break;
case 0xC2: // false
bulk_advance(1, head);
size = 1;
accepted = sax->boolean(false);
break;
case 0xC3: // true
bulk_advance(1, head);
size = 1;
accepted = sax->boolean(true);
break;
case 0xCA: // float 32
bulk_advance(5, head);
size = 5;
accepted = emit_float(load_big_endian<float>(data + 1));
break;
case 0xCB: // float 64
bulk_advance(9, head);
size = 9;
accepted = emit_float(load_big_endian<double>(data + 1));
break;
case 0xCC: // uint 8
bulk_advance(2, head);
size = 2;
accepted = emit_unsigned(data[1]);
break;
case 0xCD: // uint 16
bulk_advance(3, head);
size = 3;
accepted = emit_unsigned(load_big_endian<std::uint16_t>(data + 1));
break;
case 0xCE: // uint 32
bulk_advance(5, head);
size = 5;
accepted = emit_unsigned(load_big_endian<std::uint32_t>(data + 1));
break;
case 0xCF: // uint 64
bulk_advance(9, head);
size = 9;
accepted = emit_unsigned(load_big_endian<std::uint64_t>(data + 1));
break;
case 0xD0: // int 8
bulk_advance(2, head);
size = 2;
accepted = emit_signed(static_cast<std::int8_t>(data[1]));
break;
case 0xD1: // int 16
bulk_advance(3, head);
size = 3;
accepted = emit_signed(load_big_endian<std::int16_t>(data + 1));
break;
case 0xD2: // int 32
bulk_advance(5, head);
size = 5;
accepted = emit_signed(load_big_endian<std::int32_t>(data + 1));
break;
case 0xD3: // int 64
bulk_advance(9, head);
size = 9;
accepted = emit_signed(load_big_endian<std::int64_t>(data + 1));
break;
default:
if (head <= 0x7F)
{
// positive fixint
bulk_advance(1, head);
size = 1;
accepted = sax->number_unsigned(static_cast<number_unsigned_t>(head));
}
else if (head >= 0xE0)
{
// negative fixint
bulk_advance(1, head);
size = 1;
accepted = sax->number_integer(static_cast<std::int8_t>(head));
}
else
@@ -2952,6 +2943,10 @@ class binary_reader
}
break;
}
// as if read by get() and get_number(): current is the last byte
ia.bulk_skip(size);
chars_read += size;
current = data[size - 1];
--remaining;
if (JSON_HEDLEY_UNLIKELY(!accepted))
{
@@ -3855,7 +3850,6 @@ class binary_reader
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
// UBJSON stores numbers big-endian, BJData little-endian
const bool swap = is_little_endian != (input_format == input_format_t::bjdata);
const std::size_t start = chars_read;
std::size_t read = 0;
bool accepted = true;
while (read < count && accepted)
@@ -3867,39 +3861,16 @@ class binary_reader
byte_swap(number);
}
++read;
// where passing the value on can report an error (see
// emit_float), the position is that after the value, as if it
// was read by get_number()
set_chars_read(start + (read * size), std::integral_constant<bool, may_overflow_number_float<NumberType>()> {});
accepted = emit_ubjson_number(number);
}
// get_number() does not change current
// as if read by get_number(): current is the last byte read
ia.bulk_skip(read * size);
chars_read = start + (read * size);
chars_read += read * size;
current = data[(read * size) - 1];
remaining -= read;
return accepted;
}
/*!
@brief whether a value of type @a NumberType can be finite and still
overflow number_float_t, which @ref emit_float reports as an error
False for the default number types, so that the loop in @ref
parse_ubjson_typed_run need not track the position per value.
*/
template<typename NumberType>
static constexpr bool may_overflow_number_float() noexcept
{
return static_cast<long double>((std::numeric_limits<number_float_t>::max)()) < static_cast<long double>((std::numeric_limits<NumberType>::max)());
}
void set_chars_read(const std::size_t position, std::true_type /*track*/) noexcept
{
chars_read = position;
}
void set_chars_read(const std::size_t /*position*/, std::false_type /*track*/) const noexcept {}
template<typename NumberType, enable_if_t<std::is_floating_point<NumberType>::value, int> = 0>
bool emit_ubjson_number(const NumberType number)
{
@@ -4125,6 +4096,10 @@ class binary_reader
auto number_lexer = detail::lexer<BasicJsonType, ia_type>(detail::input_adapter(number_vector), false);
const auto result_number = number_lexer.scan();
const auto number_string = number_lexer.get_token_string();
// the number as the lexer read it, for SAX number_float(): unlike
// get_token_string(), which is meant for error messages and shortens
// long tokens, it is complete; the next scan() replaces it
const string_t number_text = number_lexer.get_string();
const auto result_remainder = number_lexer.scan();
using token_type = typename detail::lexer_base<BasicJsonType>::token_type;
@@ -4151,10 +4126,7 @@ class binary_reader
number_string,
out_of_range::create(exception_id::number_overflow, concat("number overflow parsing '", number_string, '\''), nullptr));
}
// number_string is a std::string, while the SAX interface takes a
// string_t; convert explicitly, as the two are only implicitly
// convertible for some string types
return sax->number_float(parsed_float, string_t(number_string.data(), number_string.size()));
return sax->number_float(parsed_float, number_text);
}
case token_type::uninitialized:
case token_type::literal_true:
+48 -76
View File
@@ -20605,21 +20605,6 @@ class binary_reader
return result;
}
/*!
@brief move past a value of @a size bytes, starting with @a head, that a
bulk run reads, as get() and get_number() would
Called before the value is passed on, so that an error reported while
passing it on (see @ref emit_float) has the position and the last byte
that reading the value byte by byte gives.
*/
void bulk_advance(const std::size_t size, const unsigned char head)
{
ia.bulk_skip(size);
chars_read += size;
current = head;
}
/*!
@brief read the integers, floats, booleans, and nulls that follow in a
definite-length array, without the per-element bookkeeping of
@@ -20645,70 +20630,71 @@ class binary_reader
{
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
const unsigned char head = data[0];
std::size_t size = 0;
bool accepted = false;
switch (head)
{
case 0x18:
bulk_advance(2, head);
size = 2;
accepted = emit_unsigned(data[1]);
break;
case 0x19:
bulk_advance(3, head);
size = 3;
accepted = emit_unsigned(load_big_endian<std::uint16_t>(data + 1));
break;
case 0x1A:
bulk_advance(5, head);
size = 5;
accepted = emit_unsigned(load_big_endian<std::uint32_t>(data + 1));
break;
case 0x1B:
bulk_advance(9, head);
size = 9;
accepted = emit_unsigned(load_big_endian<std::uint64_t>(data + 1));
break;
case 0x38:
bulk_advance(2, head);
size = 2;
accepted = emit_cbor_negative_integer(data[1]);
break;
case 0x39:
bulk_advance(3, head);
size = 3;
accepted = emit_cbor_negative_integer(load_big_endian<std::uint16_t>(data + 1));
break;
case 0x3A:
bulk_advance(5, head);
size = 5;
accepted = emit_cbor_negative_integer(load_big_endian<std::uint32_t>(data + 1));
break;
case 0x3B:
bulk_advance(9, head);
size = 9;
accepted = emit_cbor_negative_integer(load_big_endian<std::uint64_t>(data + 1));
break;
case 0xF4:
bulk_advance(1, head);
size = 1;
accepted = sax->boolean(false);
break;
case 0xF5:
bulk_advance(1, head);
size = 1;
accepted = sax->boolean(true);
break;
case 0xF6:
bulk_advance(1, head);
size = 1;
accepted = sax->null();
break;
case 0xFA:
bulk_advance(5, head);
size = 5;
accepted = emit_float(load_big_endian<float>(data + 1));
break;
case 0xFB:
bulk_advance(9, head);
size = 9;
accepted = emit_float(load_big_endian<double>(data + 1));
break;
default:
if (head <= 0x17)
{
bulk_advance(1, head);
size = 1;
accepted = sax->number_unsigned(static_cast<number_unsigned_t>(head));
}
else if (head >= 0x20 && head <= 0x37)
{
bulk_advance(1, head);
size = 1;
accepted = sax->number_integer(static_cast<std::int8_t>(0x20 - 1 - head));
}
else
@@ -20718,6 +20704,10 @@ class binary_reader
}
break;
}
// as if read by get() and get_number(): current is the head byte
ia.bulk_skip(size);
chars_read += size;
current = head;
--remaining;
if (JSON_HEDLEY_UNLIKELY(!accepted))
{
@@ -21555,72 +21545,73 @@ class binary_reader
{
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
const unsigned char head = data[0];
std::size_t size = 0;
bool accepted = false;
switch (head)
{
case 0xC0: // nil
bulk_advance(1, head);
size = 1;
accepted = sax->null();
break;
case 0xC2: // false
bulk_advance(1, head);
size = 1;
accepted = sax->boolean(false);
break;
case 0xC3: // true
bulk_advance(1, head);
size = 1;
accepted = sax->boolean(true);
break;
case 0xCA: // float 32
bulk_advance(5, head);
size = 5;
accepted = emit_float(load_big_endian<float>(data + 1));
break;
case 0xCB: // float 64
bulk_advance(9, head);
size = 9;
accepted = emit_float(load_big_endian<double>(data + 1));
break;
case 0xCC: // uint 8
bulk_advance(2, head);
size = 2;
accepted = emit_unsigned(data[1]);
break;
case 0xCD: // uint 16
bulk_advance(3, head);
size = 3;
accepted = emit_unsigned(load_big_endian<std::uint16_t>(data + 1));
break;
case 0xCE: // uint 32
bulk_advance(5, head);
size = 5;
accepted = emit_unsigned(load_big_endian<std::uint32_t>(data + 1));
break;
case 0xCF: // uint 64
bulk_advance(9, head);
size = 9;
accepted = emit_unsigned(load_big_endian<std::uint64_t>(data + 1));
break;
case 0xD0: // int 8
bulk_advance(2, head);
size = 2;
accepted = emit_signed(static_cast<std::int8_t>(data[1]));
break;
case 0xD1: // int 16
bulk_advance(3, head);
size = 3;
accepted = emit_signed(load_big_endian<std::int16_t>(data + 1));
break;
case 0xD2: // int 32
bulk_advance(5, head);
size = 5;
accepted = emit_signed(load_big_endian<std::int32_t>(data + 1));
break;
case 0xD3: // int 64
bulk_advance(9, head);
size = 9;
accepted = emit_signed(load_big_endian<std::int64_t>(data + 1));
break;
default:
if (head <= 0x7F)
{
// positive fixint
bulk_advance(1, head);
size = 1;
accepted = sax->number_unsigned(static_cast<number_unsigned_t>(head));
}
else if (head >= 0xE0)
{
// negative fixint
bulk_advance(1, head);
size = 1;
accepted = sax->number_integer(static_cast<std::int8_t>(head));
}
else
@@ -21630,6 +21621,10 @@ class binary_reader
}
break;
}
// as if read by get() and get_number(): current is the last byte
ia.bulk_skip(size);
chars_read += size;
current = data[size - 1];
--remaining;
if (JSON_HEDLEY_UNLIKELY(!accepted))
{
@@ -22533,7 +22528,6 @@ class binary_reader
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
// UBJSON stores numbers big-endian, BJData little-endian
const bool swap = is_little_endian != (input_format == input_format_t::bjdata);
const std::size_t start = chars_read;
std::size_t read = 0;
bool accepted = true;
while (read < count && accepted)
@@ -22545,39 +22539,16 @@ class binary_reader
byte_swap(number);
}
++read;
// where passing the value on can report an error (see
// emit_float), the position is that after the value, as if it
// was read by get_number()
set_chars_read(start + (read * size), std::integral_constant<bool, may_overflow_number_float<NumberType>()> {});
accepted = emit_ubjson_number(number);
}
// get_number() does not change current
// as if read by get_number(): current is the last byte read
ia.bulk_skip(read * size);
chars_read = start + (read * size);
chars_read += read * size;
current = data[(read * size) - 1];
remaining -= read;
return accepted;
}
/*!
@brief whether a value of type @a NumberType can be finite and still
overflow number_float_t, which @ref emit_float reports as an error
False for the default number types, so that the loop in @ref
parse_ubjson_typed_run need not track the position per value.
*/
template<typename NumberType>
static constexpr bool may_overflow_number_float() noexcept
{
return static_cast<long double>((std::numeric_limits<number_float_t>::max)()) < static_cast<long double>((std::numeric_limits<NumberType>::max)());
}
void set_chars_read(const std::size_t position, std::true_type /*track*/) noexcept
{
chars_read = position;
}
void set_chars_read(const std::size_t /*position*/, std::false_type /*track*/) const noexcept {}
template<typename NumberType, enable_if_t<std::is_floating_point<NumberType>::value, int> = 0>
bool emit_ubjson_number(const NumberType number)
{
@@ -22803,6 +22774,10 @@ class binary_reader
auto number_lexer = detail::lexer<BasicJsonType, ia_type>(detail::input_adapter(number_vector), false);
const auto result_number = number_lexer.scan();
const auto number_string = number_lexer.get_token_string();
// the number as the lexer read it, for SAX number_float(): unlike
// get_token_string(), which is meant for error messages and shortens
// long tokens, it is complete; the next scan() replaces it
const string_t number_text = number_lexer.get_string();
const auto result_remainder = number_lexer.scan();
using token_type = typename detail::lexer_base<BasicJsonType>::token_type;
@@ -22829,10 +22804,7 @@ class binary_reader
number_string,
out_of_range::create(exception_id::number_overflow, concat("number overflow parsing '", number_string, '\''), nullptr));
}
// number_string is a std::string, while the SAX interface takes a
// string_t; convert explicitly, as the two are only implicitly
// convertible for some string types
return sax->number_float(parsed_float, string_t(number_string.data(), number_string.size()));
return sax->number_float(parsed_float, number_text);
}
case token_type::uninitialized:
case token_type::literal_true:
-123
View File
@@ -14,7 +14,6 @@ using nlohmann::json;
#include <cmath>
#include <fstream>
#include <limits>
#include <list>
#include <map>
#include <string>
#include <vector>
@@ -368,125 +367,3 @@ TEST_CASE("Binary formats with narrow number types")
CHECK(format.decode(format.encode(json(1e300)), false).is_discarded());
}
}
namespace
{
// records the SAX events of narrow_json, including the arguments of parse_error
struct narrow_event_recorder
{
using number_integer_t = narrow_json::number_integer_t;
using number_unsigned_t = narrow_json::number_unsigned_t;
using number_float_t = narrow_json::number_float_t;
using string_t = narrow_json::string_t;
using binary_t = narrow_json::binary_t;
bool null()
{
events.emplace_back("null");
return true;
}
bool boolean(bool val)
{
events.emplace_back(val ? "true" : "false");
return true;
}
bool number_integer(number_integer_t val)
{
events.push_back(std::to_string(val));
return true;
}
bool number_unsigned(number_unsigned_t val)
{
events.push_back(std::to_string(val));
return true;
}
bool number_float(number_float_t val, const string_t& /*unused*/)
{
events.push_back(std::to_string(val));
return true;
}
bool string(string_t& val)
{
events.push_back(val);
return true;
}
bool binary(binary_t& /*unused*/)
{
events.emplace_back("binary");
return true;
}
bool start_object(std::size_t /*unused*/)
{
events.emplace_back("{");
return true;
}
bool key(string_t& val)
{
events.push_back(val);
return true;
}
bool end_object()
{
events.emplace_back("}");
return true;
}
bool start_array(std::size_t /*unused*/)
{
events.emplace_back("[");
return true;
}
bool end_array()
{
events.emplace_back("]");
return true;
}
bool parse_error(std::size_t position, const std::string& last_token, const nlohmann::detail::exception& ex)
{
events.push_back("parse_error(" + std::to_string(position) + ", " + last_token + ", " + ex.what() + ")");
return false;
}
std::vector<std::string> events {}; // NOLINT(readability-redundant-member-init)
};
} // namespace
TEST_CASE("Binary formats with narrow number types from contiguous and non-contiguous input")
{
// contiguous input reads runs of numbers in one loop; a number that
// overflows number_float_t must be reported with the same position and
// last byte as when the input is read byte by byte
const json values = {1, 70000, -70000, 1.5, 2.5, 1e300, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5};
const json doubles = {1.5, 2.5, 3.5, 1e300, 4.5, 5.5, 6.5, 7.5, 8.5};
struct binary_input
{
const char* name;
json::input_format_t format;
bytes data;
};
const std::vector<binary_input> inputs =
{
{"CBOR", json::input_format_t::cbor, json::to_cbor(values)},
{"CBOR doubles", json::input_format_t::cbor, json::to_cbor(doubles)},
{"MessagePack", json::input_format_t::msgpack, json::to_msgpack(values)},
{"MessagePack doubles", json::input_format_t::msgpack, json::to_msgpack(doubles)},
{"UBJSON", json::input_format_t::ubjson, json::to_ubjson(values, true)},
{"UBJSON typed doubles", json::input_format_t::ubjson, json::to_ubjson(doubles, true, true)},
{"BJData", json::input_format_t::bjdata, json::to_bjdata(values, true)},
{"BJData typed doubles", json::input_format_t::bjdata, json::to_bjdata(doubles, true, true)},
};
for (const auto& input : inputs)
{
const std::string name = input.name;
CAPTURE(name)
const std::list<std::uint8_t> bytewise(input.data.begin(), input.data.end());
narrow_event_recorder from_contiguous;
narrow_event_recorder from_bytewise;
CHECK_FALSE(narrow_json::sax_parse(input.data, &from_contiguous, input.format));
CHECK_FALSE(narrow_json::sax_parse(bytewise.begin(), bytewise.end(), &from_bytewise, input.format));
CHECK(from_contiguous.events == from_bytewise.events);
REQUIRE(!from_contiguous.events.empty());
CHECK(from_contiguous.events.back().find("out_of_range.406") != std::string::npos);
}
}
+19
View File
@@ -1278,6 +1278,25 @@ TEST_CASE("BJData")
CHECK(j.dump() == "3.141592653589793");
}
SECTION("long floating-point number")
{
// the SAX parser receives the number text as it is in the
// input, also when it is longer than the part of a token
// that error messages show
const std::vector<std::size_t> lengths = {8192, 8193, 10000};
for (const auto length : lengths)
{
CAPTURE(length)
const std::string number = "1." + std::string(length - 2, '3');
std::vector<std::uint8_t> vec = {'H', 'I', static_cast<std::uint8_t>(length & 0xFF), static_cast<std::uint8_t>(length >> 8)};
vec.insert(vec.end(), number.begin(), number.end());
SaxEventLogger logger;
CHECK(json::sax_parse(vec, &logger, json::input_format_t::bjdata));
CHECK(logger.events == std::vector<std::string>({"number_float(" + number + ")"}));
CHECK(json::from_bjdata(vec).is_number_float());
}
}
SECTION("errors")
{
// error while parsing length
+19
View File
@@ -793,6 +793,25 @@ TEST_CASE("UBJSON")
CHECK(j.dump() == "3.141592653589793");
}
SECTION("long floating-point number")
{
// the SAX parser receives the number text as it is in the
// input, also when it is longer than the part of a token
// that error messages show
const std::vector<std::size_t> lengths = {8192, 8193, 10000};
for (const auto length : lengths)
{
CAPTURE(length)
const std::string number = "1." + std::string(length - 2, '3');
std::vector<std::uint8_t> vec = {'H', 'I', static_cast<std::uint8_t>(length >> 8), static_cast<std::uint8_t>(length & 0xFF)};
vec.insert(vec.end(), number.begin(), number.end());
SaxEventLogger logger;
CHECK(json::sax_parse(vec, &logger, json::input_format_t::ubjson));
CHECK(logger.events == std::vector<std::string>({"number_float(" + number + ")"}));
CHECK(json::from_ubjson(vec).is_number_float());
}
}
SECTION("errors")
{
// error while parsing length