Compare commits

..
Author SHA1 Message Date
Niels Lohmann cc749ac699 Keep full byte2 x byte3 combinatorics in the wrong-fourth-byte sections
The maintainer wants exhaustive coverage of every byte combination here
rather than the representative-prefix reduction, matching the style of
the sibling "wrong second/third byte" sections in the same files.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>
2026-09-05 23:18:51 +02:00
Niels Lohmann 1c6ca81b8a Fix dead ill-formed-fourth-byte UTF-8 test sections (byte3/byte4 typo)
The "ill-formed: wrong fourth byte" SECTIONs in unit-unicode3.cpp,
unit-unicode4.cpp, and unit-unicode5.cpp guarded their loop with a check
on byte3 instead of byte4. Since the enclosing loop already restricts
byte3 to its valid range, the guard was always true and the section's
"continue" fired unconditionally, so check_utf8string()/check_utf8dump()
were never actually invoked for a malformed fourth byte.

Fixing the guard naively (byte3 -> byte4) would also have swept the full
byte2 x byte3 combinatorics for every byte4 value, adding millions of
redundant iterations: the lexer validates continuation bytes strictly in
sequence with early exit (see next_byte_in_range() in lexer.hpp), so once
byte2/byte3 are within their valid range, the byte4 outcome does not
depend on which valid byte2/byte3 values were chosen. Instead, byte2 and
byte3 are now held to a small hedge of representative valid prefixes
(range corners plus a midpoint) while byte4 is still swept exhaustively
over its full 0x00-0xFF range, since that is the actual property under
test. Also fixed the garbled "skip fourth second byte" comment in
unit-unicode3.cpp.

Verified offline: before the fix, the "wrong fourth byte" subcase
executes 0 assertions in all three files (proving it was dead code);
after the fix, it executes 11520 (unicode3), 34560 (unicode4), and 11520
(unicode5) assertions, and a deliberately reintroduced bug in the
lexer's byte4 range check causes it to fail (proving it is now
meaningful). Total per-file assertion counts grow by the same small
amounts, not by millions, and all other sections in these files still
pass unchanged.

Fixes #5416

Signed-off-by: Niels Lohmann <mail@nlohmann.me>
2026-09-05 23:05:42 +02:00
6 changed files with 4 additions and 74 deletions
-14
View File
@@ -748,20 +748,6 @@ class json_pointer
}
}
// the reference token consists only of digits at this point (cf. checks
// above); however, its numeric value might not be representable, in which
// case array_index() would throw out_of_range.404/410 -- contains() must
// not throw (see #5395), so such a reference token is treated as "not found"
errno = 0; // strtoull() does not reset errno on success
char* p_end = nullptr; // NOLINT(misc-const-correctness)
const unsigned long long magnitude = std::strtoull(reference_token.c_str(), &p_end, 10); // NOLINT(runtime/int)
if (JSON_HEDLEY_UNLIKELY(errno == ERANGE // the value exceeds ULLONG_MAX
|| magnitude >= static_cast<unsigned long long>((std::numeric_limits<typename BasicJsonType::size_type>::max)()))) // NOLINT(runtime/int)
{
// the array index cannot be represented as size_type
return false;
}
const auto idx = array_index<BasicJsonType>(reference_token);
if (idx >= ptr->size())
{
-14
View File
@@ -16394,20 +16394,6 @@ class json_pointer
}
}
// the reference token consists only of digits at this point (cf. checks
// above); however, its numeric value might not be representable, in which
// case array_index() would throw out_of_range.404/410 -- contains() must
// not throw (see #5395), so such a reference token is treated as "not found"
errno = 0; // strtoull() does not reset errno on success
char* p_end = nullptr; // NOLINT(misc-const-correctness)
const unsigned long long magnitude = std::strtoull(reference_token.c_str(), &p_end, 10); // NOLINT(runtime/int)
if (JSON_HEDLEY_UNLIKELY(errno == ERANGE // the value exceeds ULLONG_MAX
|| magnitude >= static_cast<unsigned long long>((std::numeric_limits<typename BasicJsonType::size_type>::max)()))) // NOLINT(runtime/int)
{
// the array index cannot be represented as size_type
return false;
}
const auto idx = array_index<BasicJsonType>(reference_token);
if (idx >= ptr->size())
{
-42
View File
@@ -319,44 +319,6 @@ TEST_CASE("JSON pointers")
CHECK_THROWS_WITH_AS(j[jp] = 1, throw_msg.c_str(), json::out_of_range&);
CHECK_THROWS_WITH_AS(j_const[jp] == 1, throw_msg.c_str(), json::out_of_range&);
// #5395: contains() must not throw for a reference token that is a
// syntactically valid array index but numerically exceeds ULLONG_MAX
// (causing strtoull() to set errno to ERANGE) -- it should just report
// that the pointer does not resolve to an element
CHECK(!j.contains(jp));
CHECK(!j_const.contains(jp));
}
{
// #5395: same as above, but using the exact reproduction from the issue
json::json_pointer const jp("/99999999999999999999");
std::string const throw_msg = "[json.exception.out_of_range.404] unresolved reference token '99999999999999999999'";
CHECK_THROWS_WITH_AS(j[jp] = 1, throw_msg.c_str(), json::out_of_range&);
CHECK_THROWS_WITH_AS(j_const[jp] == 1, throw_msg.c_str(), json::out_of_range&);
CHECK_THROWS_WITH_AS(j.at(jp) = 1, throw_msg.c_str(), json::out_of_range&);
CHECK_THROWS_WITH_AS(j_const.at(jp) == 1, throw_msg.c_str(), json::out_of_range&);
CHECK(!j.contains(jp));
CHECK(!j_const.contains(jp));
}
{
// #5395: a reference token that is numerically representable in
// unsigned long long but exceeds size_type's max (e.g. ULLONG_MAX
// itself on typical 64-bit platforms, where size_type's max equals
// ULLONG_MAX) must not make contains() throw either
json::json_pointer const jp("/18446744073709551615");
std::string const throw_msg = "[json.exception.out_of_range.410] array index 18446744073709551615 exceeds size_type";
CHECK_THROWS_WITH_AS(j[jp] = 1, throw_msg.c_str(), json::out_of_range&);
CHECK_THROWS_WITH_AS(j_const[jp] == 1, throw_msg.c_str(), json::out_of_range&);
CHECK_THROWS_WITH_AS(j.at(jp) = 1, throw_msg.c_str(), json::out_of_range&);
CHECK_THROWS_WITH_AS(j_const.at(jp) == 1, throw_msg.c_str(), json::out_of_range&);
CHECK(!j.contains(jp));
CHECK(!j_const.contains(jp));
}
// on some machines, the check below is not constant
@@ -372,10 +334,6 @@ TEST_CASE("JSON pointers")
CHECK_THROWS_WITH_AS(j[jp] = 1, throw_msg.c_str(), json::out_of_range&);
CHECK_THROWS_WITH_AS(j_const[jp] == 1, throw_msg.c_str(), json::out_of_range&);
// #5395: contains() must not throw for a reference token exceeding size_type's max
CHECK(!j.contains(jp));
CHECK(!j_const.contains(jp));
}
DOCTEST_MSVC_SUPPRESS_WARNING_POP
+2 -2
View File
@@ -304,8 +304,8 @@ TEST_CASE("Unicode (3/5)" * doctest::skip())
{
for (int byte4 = 0x00; byte4 <= 0xFF; ++byte4)
{
// skip fourth second byte
if (0x80 <= byte3 && byte3 <= 0xBF)
// skip correct fourth byte
if (0x80 <= byte4 && byte4 <= 0xBF)
{
continue;
}
+1 -1
View File
@@ -305,7 +305,7 @@ TEST_CASE("Unicode (4/5)" * doctest::skip())
for (int byte4 = 0x00; byte4 <= 0xFF; ++byte4)
{
// skip correct fourth byte
if (0x80 <= byte3 && byte3 <= 0xBF)
if (0x80 <= byte4 && byte4 <= 0xBF)
{
continue;
}
+1 -1
View File
@@ -305,7 +305,7 @@ TEST_CASE("Unicode (5/5)" * doctest::skip())
for (int byte4 = 0x00; byte4 <= 0xFF; ++byte4)
{
// skip correct fourth byte
if (0x80 <= byte3 && byte3 <= 0xBF)
if (0x80 <= byte4 && byte4 <= 0xBF)
{
continue;
}