Compare commits

..
Author SHA1 Message Date
Niels Lohmann 0395ad0992 Pass the complete text of a UBJSON/BJData high-precision number to SAX
For a high-precision number that is a floating-point number, the
UBJSON/BJData reader passed lexer::get_token_string() as the source
text to SAX number_float(). That helper is meant for error messages and
shows a token of more than 8192 characters only by its first and last
4096 characters around "...", so a SAX handler received a shortened
string for a long number.

Pass the number text the lexer collected while scanning the number
(lexer::get_string()) instead. It is complete and, as before, holds
only the number, without any whitespace or byte order mark around it
in the payload. Error messages still use get_token_string().

Signed-off-by: Niels Lohmann <mail@nlohmann.me>
2026-11-16 08:27:07 +01:00
7 changed files with 53 additions and 224 deletions

No files matched your search

+1 -1
View File
@@ -140,7 +140,7 @@ Exceptions have ids 1xx.
This error indicates a syntax error while deserializing a JSON text. The error message describes that an unexpected token (character) was encountered, and the member `byte` indicates the error position. This error indicates a syntax error while deserializing a JSON text. The error message describes that an unexpected token (character) was encountered, and the member `byte` indicates the error position.
The message ends with the last read token (`last read: '...'`), with control characters written as `<U+XXXX>`. A token of more than 8192 bytes is shown by its first and its last 4096 bytes, separated by `...`; each part is shortened by up to three bytes so that it does not end or begin within a UTF-8 character. The message ends with the last read token (`last read: '...'`), with control characters written as `<U+XXXX>`. A token of more than 8192 characters is shown by its first and its last 4096 characters, separated by `...`.
!!! failure "Example message" !!! failure "Example message"
@@ -4096,6 +4096,10 @@ class binary_reader
auto number_lexer = detail::lexer<BasicJsonType, ia_type>(detail::input_adapter(number_vector), false); auto number_lexer = detail::lexer<BasicJsonType, ia_type>(detail::input_adapter(number_vector), false);
const auto result_number = number_lexer.scan(); const auto result_number = number_lexer.scan();
const auto number_string = number_lexer.get_token_string(); const auto number_string = number_lexer.get_token_string();
// the number as the lexer read it, for SAX number_float(): unlike
// get_token_string(), which is meant for error messages and shortens
// long tokens, it is complete; the next scan() replaces it
const string_t number_text = number_lexer.get_string();
const auto result_remainder = number_lexer.scan(); const auto result_remainder = number_lexer.scan();
using token_type = typename detail::lexer_base<BasicJsonType>::token_type; using token_type = typename detail::lexer_base<BasicJsonType>::token_type;
@@ -4122,10 +4126,7 @@ class binary_reader
number_string, number_string,
out_of_range::create(exception_id::number_overflow, concat("number overflow parsing '", number_string, '\''), nullptr)); out_of_range::create(exception_id::number_overflow, concat("number overflow parsing '", number_string, '\''), nullptr));
} }
// number_string is a std::string, while the SAX interface takes a return sax->number_float(parsed_float, number_text);
// string_t; convert explicitly, as the two are only implicitly
// convertible for some string types
return sax->number_float(parsed_float, string_t(number_string.data(), number_string.size()));
} }
case token_type::uninitialized: case token_type::uninitialized:
case token_type::literal_true: case token_type::literal_true:
+2 -56
View File
@@ -2279,14 +2279,9 @@ scan_number_done:
std::string result; std::string result;
if (chars.size() > 2 * shown) if (chars.size() > 2 * shown)
{ {
// the lexer reads bytes (wide characters arrive as UTF-8), so append_escaped(result, chars.begin(), chars.begin() + static_cast<std::ptrdiff_t>(shown));
// move both cuts to character boundaries to keep the message
// valid UTF-8; this only shortens the two parts
const auto head_end = chars.begin() + static_cast<std::ptrdiff_t>(length_without_partial_utf8_at_end(chars.data(), shown));
const auto tail_begin = chars.end() - static_cast<std::ptrdiff_t>(shown);
append_escaped(result, chars.begin(), head_end);
result += "..."; result += "...";
append_escaped(result, tail_begin + static_cast<std::ptrdiff_t>(partial_utf8_length_at_start(&*tail_begin)), chars.end()); append_escaped(result, chars.end() - static_cast<std::ptrdiff_t>(shown), chars.end());
} }
else else
{ {
@@ -2296,55 +2291,6 @@ scan_number_done:
} }
private: private:
/// whether @a c is a UTF-8 continuation byte
static bool is_utf8_continuation(const char_type c) noexcept
{
return (static_cast<unsigned char>(c) & 0xC0u) == 0x80u;
}
/*!
@brief the length of the first @a n characters at @a data without a UTF-8
character that begins there but does not end there
Looks only at the last 4 of these characters: in valid UTF-8, at most 3
bytes of a character can be before the cut.
*/
static std::size_t length_without_partial_utf8_at_end(const char_type* data, const std::size_t n) noexcept
{
std::size_t lead = n - 1;
while (lead > n - 4 && is_utf8_continuation(data[lead]))
{
--lead;
}
const auto c = static_cast<unsigned char>(data[lead]);
std::size_t length = 1;
if (c >= 0xF0u)
{
length = 4;
}
else if (c >= 0xE0u)
{
length = 3;
}
else if (c >= 0xC0u)
{
length = 2;
}
return lead + length > n ? lead : n;
}
/// the number of UTF-8 continuation bytes (at most 3) at the start of @a data, which
/// belong to a character that begins before it
static std::size_t partial_utf8_length_at_start(const char_type* data) noexcept
{
std::size_t skip = 0;
while (skip < 3 && is_utf8_continuation(data[skip]))
{
++skip;
}
return skip;
}
/// append the characters [first, last) to @a result, escaping control characters /// append the characters [first, last) to @a result, escaping control characters
template<typename Iterator> template<typename Iterator>
static void append_escaped(std::string& result, Iterator first, Iterator last) static void append_escaped(std::string& result, Iterator first, Iterator last)
+7 -60
View File
@@ -15782,14 +15782,9 @@ scan_number_done:
std::string result; std::string result;
if (chars.size() > 2 * shown) if (chars.size() > 2 * shown)
{ {
// the lexer reads bytes (wide characters arrive as UTF-8), so append_escaped(result, chars.begin(), chars.begin() + static_cast<std::ptrdiff_t>(shown));
// move both cuts to character boundaries to keep the message
// valid UTF-8; this only shortens the two parts
const auto head_end = chars.begin() + static_cast<std::ptrdiff_t>(length_without_partial_utf8_at_end(chars.data(), shown));
const auto tail_begin = chars.end() - static_cast<std::ptrdiff_t>(shown);
append_escaped(result, chars.begin(), head_end);
result += "..."; result += "...";
append_escaped(result, tail_begin + static_cast<std::ptrdiff_t>(partial_utf8_length_at_start(&*tail_begin)), chars.end()); append_escaped(result, chars.end() - static_cast<std::ptrdiff_t>(shown), chars.end());
} }
else else
{ {
@@ -15799,55 +15794,6 @@ scan_number_done:
} }
private: private:
/// whether @a c is a UTF-8 continuation byte
static bool is_utf8_continuation(const char_type c) noexcept
{
return (static_cast<unsigned char>(c) & 0xC0u) == 0x80u;
}
/*!
@brief the length of the first @a n characters at @a data without a UTF-8
character that begins there but does not end there
Looks only at the last 4 of these characters: in valid UTF-8, at most 3
bytes of a character can be before the cut.
*/
static std::size_t length_without_partial_utf8_at_end(const char_type* data, const std::size_t n) noexcept
{
std::size_t lead = n - 1;
while (lead > n - 4 && is_utf8_continuation(data[lead]))
{
--lead;
}
const auto c = static_cast<unsigned char>(data[lead]);
std::size_t length = 1;
if (c >= 0xF0u)
{
length = 4;
}
else if (c >= 0xE0u)
{
length = 3;
}
else if (c >= 0xC0u)
{
length = 2;
}
return lead + length > n ? lead : n;
}
/// the number of UTF-8 continuation bytes (at most 3) at the start of @a data, which
/// belong to a character that begins before it
static std::size_t partial_utf8_length_at_start(const char_type* data) noexcept
{
std::size_t skip = 0;
while (skip < 3 && is_utf8_continuation(data[skip]))
{
++skip;
}
return skip;
}
/// append the characters [first, last) to @a result, escaping control characters /// append the characters [first, last) to @a result, escaping control characters
template<typename Iterator> template<typename Iterator>
static void append_escaped(std::string& result, Iterator first, Iterator last) static void append_escaped(std::string& result, Iterator first, Iterator last)
@@ -22828,6 +22774,10 @@ class binary_reader
auto number_lexer = detail::lexer<BasicJsonType, ia_type>(detail::input_adapter(number_vector), false); auto number_lexer = detail::lexer<BasicJsonType, ia_type>(detail::input_adapter(number_vector), false);
const auto result_number = number_lexer.scan(); const auto result_number = number_lexer.scan();
const auto number_string = number_lexer.get_token_string(); const auto number_string = number_lexer.get_token_string();
// the number as the lexer read it, for SAX number_float(): unlike
// get_token_string(), which is meant for error messages and shortens
// long tokens, it is complete; the next scan() replaces it
const string_t number_text = number_lexer.get_string();
const auto result_remainder = number_lexer.scan(); const auto result_remainder = number_lexer.scan();
using token_type = typename detail::lexer_base<BasicJsonType>::token_type; using token_type = typename detail::lexer_base<BasicJsonType>::token_type;
@@ -22854,10 +22804,7 @@ class binary_reader
number_string, number_string,
out_of_range::create(exception_id::number_overflow, concat("number overflow parsing '", number_string, '\''), nullptr)); out_of_range::create(exception_id::number_overflow, concat("number overflow parsing '", number_string, '\''), nullptr));
} }
// number_string is a std::string, while the SAX interface takes a return sax->number_float(parsed_float, number_text);
// string_t; convert explicitly, as the two are only implicitly
// convertible for some string types
return sax->number_float(parsed_float, string_t(number_string.data(), number_string.size()));
} }
case token_type::uninitialized: case token_type::uninitialized:
case token_type::literal_true: case token_type::literal_true:
+19
View File
@@ -1278,6 +1278,25 @@ TEST_CASE("BJData")
CHECK(j.dump() == "3.141592653589793"); CHECK(j.dump() == "3.141592653589793");
} }
SECTION("long floating-point number")
{
// the SAX parser receives the number text as it is in the
// input, also when it is longer than the part of a token
// that error messages show
const std::vector<std::size_t> lengths = {8192, 8193, 10000};
for (const auto length : lengths)
{
CAPTURE(length)
const std::string number = "1." + std::string(length - 2, '3');
std::vector<std::uint8_t> vec = {'H', 'I', static_cast<std::uint8_t>(length & 0xFF), static_cast<std::uint8_t>(length >> 8)};
vec.insert(vec.end(), number.begin(), number.end());
SaxEventLogger logger;
CHECK(json::sax_parse(vec, &logger, json::input_format_t::bjdata));
CHECK(logger.events == std::vector<std::string>({"number_float(" + number + ")"}));
CHECK(json::from_bjdata(vec).is_number_float());
}
}
SECTION("errors") SECTION("errors")
{ {
// error while parsing length // error while parsing length
-103
View File
@@ -3380,108 +3380,5 @@ TEST_CASE("parse error message of a long token")
CHECK(message(text, kind) == "no error"); CHECK(message(text, kind) == "no error");
} }
} }
SECTION("a long string of multi-byte characters")
{
// the parts of a long token are cut at character boundaries, so the
// message of valid UTF-8 input is valid UTF-8, also for wide strings
const auto last_read = [](const std::string & m) -> std::string
{
const auto pos = m.find("last read: '");
return pos == std::string::npos ? std::string() : m.substr(pos + 12, m.size() - pos - 13);
};
const auto utf8 = [](const char32_t c) -> std::string
{
std::string result;
if (c < 0x800)
{
result.push_back(static_cast<char>(0xC0 | (c >> 6)));
}
else if (c < 0x10000)
{
result.push_back(static_cast<char>(0xE0 | (c >> 12)));
result.push_back(static_cast<char>(0x80 | ((c >> 6) & 0x3F)));
}
else
{
result.push_back(static_cast<char>(0xF0 | (c >> 18)));
result.push_back(static_cast<char>(0x80 | ((c >> 12) & 0x3F)));
result.push_back(static_cast<char>(0x80 | ((c >> 6) & 0x3F)));
}
result.push_back(static_cast<char>(0x80 | (c & 0x3F)));
return result;
};
// characters of 2, 3, and 4 bytes
const std::vector<char32_t> characters = {0xE9, 0x20AC, 0x1F600};
for (const char32_t c : characters)
{
for (std::size_t offset = 0; offset < 4; ++offset)
{
CAPTURE(static_cast<std::uint32_t>(c))
CAPTURE(offset)
// an unterminated string: the token is the whole input
std::string text = "\"" + std::string(offset, 'a');
std::u32string text32 = U"\"" + std::u32string(offset, U'a');
while (text.size() < 9000)
{
text += utf8(c);
text32.push_back(c);
}
std::u16string text16;
for (const char32_t c32 : text32)
{
if (c32 < 0x10000)
{
text16.push_back(static_cast<char16_t>(c32));
}
else
{
text16.push_back(static_cast<char16_t>(0xD800 + ((c32 - 0x10000) >> 10)));
text16.push_back(static_cast<char16_t>(0xDC00 + ((c32 - 0x10000) & 0x3FF)));
}
}
const std::string shown = last_read(message(text, 0));
const auto dots = shown.find("...");
REQUIRE(dots != std::string::npos);
// the first and the last at most 4096 bytes, without parts of characters
CHECK(dots <= 4096);
CHECK(dots > 4092);
CHECK(text.compare(0, dots, shown, 0, dots) == 0);
const std::size_t tail = shown.size() - dots - 3;
CHECK(tail <= 4096);
CHECK(tail > 4092);
CHECK(text.compare(text.size() - tail, tail, shown, dots + 3, tail) == 0);
CHECK_NOTHROW(json(shown).dump());
for (int kind = 1; kind < 4; ++kind)
{
CAPTURE(kind)
CHECK(last_read(message(text, kind)) == shown);
}
json _;
std::string m32;
std::string m16;
try
{
_ = json::parse(text32);
}
catch (const json::parse_error& e)
{
m32 = e.what();
}
try
{
_ = json::parse(text16);
}
catch (const json::parse_error& e)
{
m16 = e.what();
}
CHECK(last_read(m32) == shown);
CHECK(last_read(m16) == shown);
}
}
}
} }
#endif #endif
+19
View File
@@ -793,6 +793,25 @@ TEST_CASE("UBJSON")
CHECK(j.dump() == "3.141592653589793"); CHECK(j.dump() == "3.141592653589793");
} }
SECTION("long floating-point number")
{
// the SAX parser receives the number text as it is in the
// input, also when it is longer than the part of a token
// that error messages show
const std::vector<std::size_t> lengths = {8192, 8193, 10000};
for (const auto length : lengths)
{
CAPTURE(length)
const std::string number = "1." + std::string(length - 2, '3');
std::vector<std::uint8_t> vec = {'H', 'I', static_cast<std::uint8_t>(length >> 8), static_cast<std::uint8_t>(length & 0xFF)};
vec.insert(vec.end(), number.begin(), number.end());
SaxEventLogger logger;
CHECK(json::sax_parse(vec, &logger, json::input_format_t::ubjson));
CHECK(logger.events == std::vector<std::string>({"number_float(" + number + ")"}));
CHECK(json::from_ubjson(vec).is_number_float());
}
}
SECTION("errors") SECTION("errors")
{ {
// error while parsing length // error while parsing length