Cut long tokens in parse error messages at UTF-8 character boundaries

A parse error message shows a token of more than 8192 bytes by its
first and its last 4096 bytes around "...". The cuts were at fixed byte
offsets, so they could split a UTF-8 character: for valid UTF-8 input,
what() and the last token passed to SAX parse_error() were not valid
UTF-8, and storing the message in a json value and dumping it threw
type_error.316.

Move the end of the first part back and the start of the last part
forward to a character boundary, by at most three bytes each. The
lexer always reads bytes (wide strings are converted to UTF-8 first),
and the decision depends only on bytes inside the two parts, so the
message stays the same for every kind of input. Both parts only get
shorter, so the buffers that keep the token need no more room.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>
This commit is contained in:
Niels Lohmann committed 2026-11-16 08:12:02 +01:00
1 parent abbc29963a
commit 04fbced73d
4 files changed
+216 -5

No files matched your search

+103
View File
@@ -3380,5 +3380,108 @@ TEST_CASE("parse error message of a long token")
CHECK(message(text, kind) == "no error");
}
}
SECTION("a long string of multi-byte characters")
{
// the parts of a long token are cut at character boundaries, so the
// message of valid UTF-8 input is valid UTF-8, also for wide strings
const auto last_read = [](const std::string & m) -> std::string
{
const auto pos = m.find("last read: '");
return pos == std::string::npos ? std::string() : m.substr(pos + 12, m.size() - pos - 13);
};
const auto utf8 = [](const char32_t c) -> std::string
{
std::string result;
if (c < 0x800)
{
result.push_back(static_cast<char>(0xC0 | (c >> 6)));
}
else if (c < 0x10000)
{
result.push_back(static_cast<char>(0xE0 | (c >> 12)));
result.push_back(static_cast<char>(0x80 | ((c >> 6) & 0x3F)));
}
else
{
result.push_back(static_cast<char>(0xF0 | (c >> 18)));
result.push_back(static_cast<char>(0x80 | ((c >> 12) & 0x3F)));
result.push_back(static_cast<char>(0x80 | ((c >> 6) & 0x3F)));
}
result.push_back(static_cast<char>(0x80 | (c & 0x3F)));
return result;
};
// characters of 2, 3, and 4 bytes
const std::vector<char32_t> characters = {0xE9, 0x20AC, 0x1F600};
for (const char32_t c : characters)
{
for (std::size_t offset = 0; offset < 4; ++offset)
{
CAPTURE(static_cast<std::uint32_t>(c))
CAPTURE(offset)
// an unterminated string: the token is the whole input
std::string text = "\"" + std::string(offset, 'a');
std::u32string text32 = U"\"" + std::u32string(offset, U'a');
while (text.size() < 9000)
{
text += utf8(c);
text32.push_back(c);
}
std::u16string text16;
for (const char32_t c32 : text32)
{
if (c32 < 0x10000)
{
text16.push_back(static_cast<char16_t>(c32));
}
else
{
text16.push_back(static_cast<char16_t>(0xD800 + ((c32 - 0x10000) >> 10)));
text16.push_back(static_cast<char16_t>(0xDC00 + ((c32 - 0x10000) & 0x3FF)));
}
}
const std::string shown = last_read(message(text, 0));
const auto dots = shown.find("...");
REQUIRE(dots != std::string::npos);
// the first and the last at most 4096 bytes, without parts of characters
CHECK(dots <= 4096);
CHECK(dots > 4092);
CHECK(text.compare(0, dots, shown, 0, dots) == 0);
const std::size_t tail = shown.size() - dots - 3;
CHECK(tail <= 4096);
CHECK(tail > 4092);
CHECK(text.compare(text.size() - tail, tail, shown, dots + 3, tail) == 0);
CHECK_NOTHROW(json(shown).dump());
for (int kind = 1; kind < 4; ++kind)
{
CAPTURE(kind)
CHECK(last_read(message(text, kind)) == shown);
}
json _;
std::string m32;
std::string m16;
try
{
_ = json::parse(text32);
}
catch (const json::parse_error& e)
{
m32 = e.what();
}
try
{
_ = json::parse(text16);
}
catch (const json::parse_error& e)
{
m16 = e.what();
}
CHECK(last_read(m32) == shown);
CHECK(last_read(m16) == shown);
}
}
}
}
#endif