amorfo77/json
1
0
Fork 0
mirror of https://github.com/nlohmann/json.git synced 2026-09-06 00:08:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Keep full byte2 x byte3 combinatorics in the wrong-fourth-byte sections

Browse Source 
The maintainer wants exhaustive coverage of every byte combination here
rather than the representative-prefix reduction, matching the style of
the sibling "wrong second/third byte" sections in the same files.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>
This commit is contained in:
Niels Lohmann
2026-09-05 23:18:51 +02:00
parent 1c6ca81b8a
commit cc749ac699
3 changed files with 33 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/src/unit-unicode3.cpp
+11 -30
View File
@@ -296,42 +296,23 @@ TEST_CASE("Unicode (3/5)" * doctest::skip())
SECTION("ill-formed: wrong fourth byte")
{
// The lexer (see next_byte_in_range() in lexer.hpp) validates the
// continuation bytes strictly in sequence and bails out on the first
// byte that is out of range. So once byte2 and byte3 are anywhere
// inside their own valid range, whether byte4 is accepted or rejected
// depends only on byte4's value -- not on which particular valid
// byte2/byte3 combination was used to reach it. Sweeping the full
// byte2 x byte3 combinatorics here (as the other "wrong Nth byte"
// sections do for the byte they target) would therefore add a huge
// number of iterations for zero additional coverage. Instead, byte2
// and byte3 are held to a small hedge of representative valid
// prefixes -- the corners and midpoint of their valid ranges -- while
// byte4 is still swept exhaustively over 0x00-0xFF, since "byte4 out
// of range is rejected for every value it could take" is the actual
// property under test. If the UTF-8 decoder is ever reworked (e.g.
// into a table-driven/bulk scanner), this equivalence-class
// assumption should be re-audited.
static const int byte2_values[] = {0x90, 0x90, 0xBF, 0xBF, 0xA8};
static const int byte3_values[] = {0x80, 0xBF, 0x80, 0xBF, 0xA0};
for (int byte1 = 0xF0; byte1 <= 0xF0; ++byte1)
{
for (size_t idx = 0; idx < sizeof(byte2_values) / sizeof(byte2_values[0]); ++idx)
for (int byte2 = 0x90; byte2 <= 0xBF; ++byte2)
{
const int byte2 = byte2_values[idx];
const int byte3 = byte3_values[idx];
for (int byte4 = 0x00; byte4 <= 0xFF; ++byte4)
for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3)
{
// skip correct fourth byte
if (0x80 <= byte4 && byte4 <= 0xBF)
for (int byte4 = 0x00; byte4 <= 0xFF; ++byte4)
{
continue;
}
// skip correct fourth byte
if (0x80 <= byte4 && byte4 <= 0xBF)
{
continue;
}
check_utf8string(false, byte1, byte2, byte3, byte4);
check_utf8dump(false, byte1, byte2, byte3, byte4);
check_utf8string(false, byte1, byte2, byte3, byte4);
check_utf8dump(false, byte1, byte2, byte3, byte4);
}
}
}
}
tests/src/unit-unicode4.cpp
+11 -30
View File
@@ -296,42 +296,23 @@ TEST_CASE("Unicode (4/5)" * doctest::skip())
SECTION("ill-formed: wrong fourth byte")
{
// The lexer (see next_byte_in_range() in lexer.hpp) validates the
// continuation bytes strictly in sequence and bails out on the first
// byte that is out of range. So once byte2 and byte3 are anywhere
// inside their own valid range, whether byte4 is accepted or rejected
// depends only on byte4's value -- not on which particular valid
// byte2/byte3 combination was used to reach it. Sweeping the full
// byte2 x byte3 combinatorics here (as the other "wrong Nth byte"
// sections do for the byte they target) would therefore add a huge
// number of iterations for zero additional coverage. Instead, byte2
// and byte3 are held to a small hedge of representative valid
// prefixes -- the corners and midpoint of their valid ranges -- while
// byte4 is still swept exhaustively over 0x00-0xFF, since "byte4 out
// of range is rejected for every value it could take" is the actual
// property under test. If the UTF-8 decoder is ever reworked (e.g.
// into a table-driven/bulk scanner), this equivalence-class
// assumption should be re-audited.
static const int byte2_values[] = {0x80, 0x80, 0xBF, 0xBF, 0xA0};
static const int byte3_values[] = {0x80, 0xBF, 0x80, 0xBF, 0xA0};
for (int byte1 = 0xF1; byte1 <= 0xF3; ++byte1)
{
for (size_t idx = 0; idx < sizeof(byte2_values) / sizeof(byte2_values[0]); ++idx)
for (int byte2 = 0x80; byte2 <= 0xBF; ++byte2)
{
const int byte2 = byte2_values[idx];
const int byte3 = byte3_values[idx];
for (int byte4 = 0x00; byte4 <= 0xFF; ++byte4)
for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3)
{
// skip correct fourth byte
if (0x80 <= byte4 && byte4 <= 0xBF)
for (int byte4 = 0x00; byte4 <= 0xFF; ++byte4)
{
continue;
}
// skip correct fourth byte
if (0x80 <= byte4 && byte4 <= 0xBF)
{
continue;
}
check_utf8string(false, byte1, byte2, byte3, byte4);
check_utf8dump(false, byte1, byte2, byte3, byte4);
check_utf8string(false, byte1, byte2, byte3, byte4);
check_utf8dump(false, byte1, byte2, byte3, byte4);
}
}
}
}
tests/src/unit-unicode5.cpp
+11 -30
View File
@@ -296,42 +296,23 @@ TEST_CASE("Unicode (5/5)" * doctest::skip())
SECTION("ill-formed: wrong fourth byte")
{
// The lexer (see next_byte_in_range() in lexer.hpp) validates the
// continuation bytes strictly in sequence and bails out on the first
// byte that is out of range. So once byte2 and byte3 are anywhere
// inside their own valid range, whether byte4 is accepted or rejected
// depends only on byte4's value -- not on which particular valid
// byte2/byte3 combination was used to reach it. Sweeping the full
// byte2 x byte3 combinatorics here (as the other "wrong Nth byte"
// sections do for the byte they target) would therefore add a huge
// number of iterations for zero additional coverage. Instead, byte2
// and byte3 are held to a small hedge of representative valid
// prefixes -- the corners and midpoint of their valid ranges -- while
// byte4 is still swept exhaustively over 0x00-0xFF, since "byte4 out
// of range is rejected for every value it could take" is the actual
// property under test. If the UTF-8 decoder is ever reworked (e.g.
// into a table-driven/bulk scanner), this equivalence-class
// assumption should be re-audited.
static const int byte2_values[] = {0x80, 0x80, 0x8F, 0x8F, 0x88};
static const int byte3_values[] = {0x80, 0xBF, 0x80, 0xBF, 0xA0};
for (int byte1 = 0xF4; byte1 <= 0xF4; ++byte1)
{
for (size_t idx = 0; idx < sizeof(byte2_values) / sizeof(byte2_values[0]); ++idx)
for (int byte2 = 0x80; byte2 <= 0x8F; ++byte2)
{
const int byte2 = byte2_values[idx];
const int byte3 = byte3_values[idx];
for (int byte4 = 0x00; byte4 <= 0xFF; ++byte4)
for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3)
{
// skip correct fourth byte
if (0x80 <= byte4 && byte4 <= 0xBF)
for (int byte4 = 0x00; byte4 <= 0xFF; ++byte4)
{
continue;
}
// skip correct fourth byte
if (0x80 <= byte4 && byte4 <= 0xBF)
{
continue;
}
check_utf8string(false, byte1, byte2, byte3, byte4);
check_utf8dump(false, byte1, byte2, byte3, byte4);
check_utf8string(false, byte1, byte2, byte3, byte4);
check_utf8dump(false, byte1, byte2, byte3, byte4);
}
}
}
}