Compare commits

..
Author SHA1 Message Date
Niels Lohmann 9862cf25f6 Test escaped object keys decoded through NLOHMANN_DEFINE_TYPE from a json_view
A json_view or json_document decodes a macro-defined struct directly and looks
the members up by their decoded keys. Only plain keys were covered so far.

The new section decodes structs defined with NLOHMANN_DEFINE_TYPE_INTRUSIVE,
..._WITH_DEFAULT and ..._WITH_NAMES from documents whose keys contain escape
sequences, and compares the result with the conversion of the materialized
value. It also pins the difference for a duplicate key written once plain and
once escaped (the view takes the first member, the materialized value keeps
the last one), and that an escaped backslash is part of the decoded key.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>
2026-10-30 22:37:32 +01:00
4 changed files with 52 additions and 418 deletions

No files matched your search

+14 -172
View File
@@ -741,13 +741,11 @@ class binary_reader
bool get_cbor_negative_integer()
{
NumberType number{};
return get_number(number) && emit_cbor_negative_integer(number);
}
if (JSON_HEDLEY_UNLIKELY(!get_number(number)))
{
return false;
}
/// pass the CBOR negative integer -1 - @a number to the SAX parser
template<typename NumberType>
bool emit_cbor_negative_integer(const NumberType number)
{
// the value is -1 - number, which fits into number_integer_t
// whenever number does; the outer cast undoes the integral promotion
// for number_integer_t types narrower than int
@@ -972,7 +970,7 @@ class binary_reader
case 0x95:
case 0x96:
case 0x97:
return enter_cbor_array(conditional_static_cast<std::size_t>(static_cast<unsigned int>(current) & 0x1Fu));
return enter_array(conditional_static_cast<std::size_t>(static_cast<unsigned int>(current) & 0x1Fu));
case 0x98: // array (one-byte uint8_t for n follows)
case 0x99: // array (two-byte uint16_t for n follow)
@@ -981,7 +979,7 @@ class binary_reader
{
std::uint64_t len{};
std::size_t size{};
return get_cbor_argument(len) && get_cbor_container_size(len, size, "array") && enter_cbor_array(size);
return get_cbor_argument(len) && get_cbor_container_size(len, size, "array") && enter_array(size);
}
case 0x9F: // array (indefinite length)
@@ -1678,157 +1676,6 @@ class binary_reader
return true;
}
/*!
@brief read a big-endian number of type @a NumberType at @a data
*/
template<typename NumberType>
NumberType load_big_endian(const unsigned char* data) const noexcept
{
NumberType result{};
std::memcpy(&result, data, sizeof(NumberType));
if (is_little_endian)
{
byte_swap(result);
}
return result;
}
/*!
@brief read the integers, floats, booleans, and nulls that follow in a
definite-length array, without the per-element bookkeeping of
@ref parse_cbor_internal
Stops at the end of the array, at any other kind of value, and when fewer
than 9 bytes (the longest of these values) are left in the input, leaving
those to @ref parse_cbor_value, which reads them the same way.
@param[in,out] remaining the number of elements of the array not yet read
@return whether the SAX parser accepted the values
*/
bool parse_cbor_scalar_run(std::size_t& remaining)
{
while (remaining != 0 && ia.bulk_remaining() >= 9)
{
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
const unsigned char head = data[0];
std::size_t size = 0;
bool accepted = true;
switch (head)
{
case 0x18:
size = 2;
accepted = emit_unsigned(data[1]);
break;
case 0x19:
size = 3;
accepted = emit_unsigned(load_big_endian<std::uint16_t>(data + 1));
break;
case 0x1A:
size = 5;
accepted = emit_unsigned(load_big_endian<std::uint32_t>(data + 1));
break;
case 0x1B:
size = 9;
accepted = emit_unsigned(load_big_endian<std::uint64_t>(data + 1));
break;
case 0x38:
size = 2;
accepted = emit_cbor_negative_integer(data[1]);
break;
case 0x39:
size = 3;
accepted = emit_cbor_negative_integer(load_big_endian<std::uint16_t>(data + 1));
break;
case 0x3A:
size = 5;
accepted = emit_cbor_negative_integer(load_big_endian<std::uint32_t>(data + 1));
break;
case 0x3B:
size = 9;
accepted = emit_cbor_negative_integer(load_big_endian<std::uint64_t>(data + 1));
break;
case 0xF4:
size = 1;
accepted = sax->boolean(false);
break;
case 0xF5:
size = 1;
accepted = sax->boolean(true);
break;
case 0xF6:
size = 1;
accepted = sax->null();
break;
case 0xFA:
size = 5;
accepted = emit_float(load_big_endian<float>(data + 1));
break;
case 0xFB:
size = 9;
accepted = emit_float(load_big_endian<double>(data + 1));
break;
default:
if (head <= 0x17)
{
size = 1;
accepted = sax->number_unsigned(static_cast<number_unsigned_t>(head));
}
else if (head >= 0x20 && head <= 0x37)
{
size = 1;
accepted = sax->number_integer(static_cast<std::int8_t>(0x20 - 1 - head));
}
else
{
// any other value is read by parse_cbor_value
return true;
}
break;
}
// as if read by get() and get_number(): current is the head byte
ia.bulk_skip(size);
chars_read += size;
current = head;
--remaining;
if (JSON_HEDLEY_UNLIKELY(!accepted))
{
return false;
}
}
return true;
}
/*!
@brief open a definite-length array and read the scalars it begins with
(see @ref parse_cbor_scalar_run)
Only the start of an array is tried, which covers arrays of numbers
(e.g. coordinates) but costs arrays of other values only one test, not one
per element.
@param[in] len the number of elements
@return whether the SAX parser accepted the array and the values read
*/
bool enter_cbor_array(const std::size_t len)
{
return enter_array(len) && (len == 0 || parse_cbor_array_start(std::integral_constant<bool, bulk_scan> {}));
}
/// read the scalars the array just opened begins with; out of line, so
/// that @ref parse_cbor_value stays small enough to be inlined into
/// @ref parse_cbor_internal
JSON_HEDLEY_NEVER_INLINE
bool parse_cbor_array_start(std::true_type /*bulk*/)
{
return parse_cbor_scalar_run(container_stack.back().remaining);
}
/// input that is not contiguous: every value is read by parse_cbor_value
bool parse_cbor_array_start(std::false_type /*bulk*/) const noexcept
{
return true;
}
/*!
@brief read a CBOR value and everything nested inside it
@@ -1858,7 +1705,6 @@ class binary_reader
// pop_back() below, which destroys the container_stack element
// it would otherwise alias
const container_frame top = container_stack.back();
const bool is_object = top.is_object;
bool at_end = false;
if (top.remaining != npos)
@@ -1870,7 +1716,7 @@ class binary_reader
{
// claim the element about to be read
--container_stack.back().remaining;
if (is_object)
if (top.is_object)
{
get();
}
@@ -1883,7 +1729,7 @@ class binary_reader
// Testing for it consumes a byte, which is the first byte
// of the next element when it is not one.
at_end = (get() == 0xFF);
fetch = is_object;
fetch = top.is_object;
}
if (at_end)
@@ -1900,7 +1746,7 @@ class binary_reader
continue;
}
if (is_object)
if (top.is_object)
{
if (JSON_HEDLEY_UNLIKELY(!skip_cbor_key_tags(tag_handler)))
{
@@ -3845,13 +3691,13 @@ class binary_reader
return sax->null();
case 0xFB:
return sax->number_float(static_cast<number_float_t>(-1.0), no_source);
return sax->number_float(static_cast<number_float_t>(-1.0), "");
case 0xFC:
return sax->number_float(static_cast<number_float_t>(0.0), no_source);
return sax->number_float(static_cast<number_float_t>(0.0), "");
case 0xFD:
return sax->number_float(static_cast<number_float_t>(1.0), no_source);
return sax->number_float(static_cast<number_float_t>(1.0), "");
case 0xFF: // empty string
{
@@ -4355,7 +4201,7 @@ class binary_reader
{
return out_of_range_error(exception_id::number_overflow, "number overflow", "value");
}
return sax->number_float(result, no_source);
return sax->number_float(result, "");
}
/*!
@@ -4415,7 +4261,7 @@ class binary_reader
}();
return sax->number_float((half & 0x8000u) != 0
? static_cast<number_float_t>(-val)
: static_cast<number_float_t>(val), no_source);
: static_cast<number_float_t>(val), "");
}
/*!
@@ -4740,10 +4586,6 @@ class binary_reader
/// the SAX parser
json_sax_t* sax = nullptr;
/// the (empty) source text of every number_float event: binary formats
/// have none, and a string_t built for each float would cost more
const string_t no_source{};
/// the containers that have been opened and not closed yet; see @ref container_frame
std::vector<container_frame> container_stack{};
+14 -172
View File
@@ -18517,13 +18517,11 @@ class binary_reader
bool get_cbor_negative_integer()
{
NumberType number{};
return get_number(number) && emit_cbor_negative_integer(number);
}
if (JSON_HEDLEY_UNLIKELY(!get_number(number)))
{
return false;
}
/// pass the CBOR negative integer -1 - @a number to the SAX parser
template<typename NumberType>
bool emit_cbor_negative_integer(const NumberType number)
{
// the value is -1 - number, which fits into number_integer_t
// whenever number does; the outer cast undoes the integral promotion
// for number_integer_t types narrower than int
@@ -18748,7 +18746,7 @@ class binary_reader
case 0x95:
case 0x96:
case 0x97:
return enter_cbor_array(conditional_static_cast<std::size_t>(static_cast<unsigned int>(current) & 0x1Fu));
return enter_array(conditional_static_cast<std::size_t>(static_cast<unsigned int>(current) & 0x1Fu));
case 0x98: // array (one-byte uint8_t for n follows)
case 0x99: // array (two-byte uint16_t for n follow)
@@ -18757,7 +18755,7 @@ class binary_reader
{
std::uint64_t len{};
std::size_t size{};
return get_cbor_argument(len) && get_cbor_container_size(len, size, "array") && enter_cbor_array(size);
return get_cbor_argument(len) && get_cbor_container_size(len, size, "array") && enter_array(size);
}
case 0x9F: // array (indefinite length)
@@ -19454,157 +19452,6 @@ class binary_reader
return true;
}
/*!
@brief read a big-endian number of type @a NumberType at @a data
*/
template<typename NumberType>
NumberType load_big_endian(const unsigned char* data) const noexcept
{
NumberType result{};
std::memcpy(&result, data, sizeof(NumberType));
if (is_little_endian)
{
byte_swap(result);
}
return result;
}
/*!
@brief read the integers, floats, booleans, and nulls that follow in a
definite-length array, without the per-element bookkeeping of
@ref parse_cbor_internal
Stops at the end of the array, at any other kind of value, and when fewer
than 9 bytes (the longest of these values) are left in the input, leaving
those to @ref parse_cbor_value, which reads them the same way.
@param[in,out] remaining the number of elements of the array not yet read
@return whether the SAX parser accepted the values
*/
bool parse_cbor_scalar_run(std::size_t& remaining)
{
while (remaining != 0 && ia.bulk_remaining() >= 9)
{
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
const unsigned char head = data[0];
std::size_t size = 0;
bool accepted = true;
switch (head)
{
case 0x18:
size = 2;
accepted = emit_unsigned(data[1]);
break;
case 0x19:
size = 3;
accepted = emit_unsigned(load_big_endian<std::uint16_t>(data + 1));
break;
case 0x1A:
size = 5;
accepted = emit_unsigned(load_big_endian<std::uint32_t>(data + 1));
break;
case 0x1B:
size = 9;
accepted = emit_unsigned(load_big_endian<std::uint64_t>(data + 1));
break;
case 0x38:
size = 2;
accepted = emit_cbor_negative_integer(data[1]);
break;
case 0x39:
size = 3;
accepted = emit_cbor_negative_integer(load_big_endian<std::uint16_t>(data + 1));
break;
case 0x3A:
size = 5;
accepted = emit_cbor_negative_integer(load_big_endian<std::uint32_t>(data + 1));
break;
case 0x3B:
size = 9;
accepted = emit_cbor_negative_integer(load_big_endian<std::uint64_t>(data + 1));
break;
case 0xF4:
size = 1;
accepted = sax->boolean(false);
break;
case 0xF5:
size = 1;
accepted = sax->boolean(true);
break;
case 0xF6:
size = 1;
accepted = sax->null();
break;
case 0xFA:
size = 5;
accepted = emit_float(load_big_endian<float>(data + 1));
break;
case 0xFB:
size = 9;
accepted = emit_float(load_big_endian<double>(data + 1));
break;
default:
if (head <= 0x17)
{
size = 1;
accepted = sax->number_unsigned(static_cast<number_unsigned_t>(head));
}
else if (head >= 0x20 && head <= 0x37)
{
size = 1;
accepted = sax->number_integer(static_cast<std::int8_t>(0x20 - 1 - head));
}
else
{
// any other value is read by parse_cbor_value
return true;
}
break;
}
// as if read by get() and get_number(): current is the head byte
ia.bulk_skip(size);
chars_read += size;
current = head;
--remaining;
if (JSON_HEDLEY_UNLIKELY(!accepted))
{
return false;
}
}
return true;
}
/*!
@brief open a definite-length array and read the scalars it begins with
(see @ref parse_cbor_scalar_run)
Only the start of an array is tried, which covers arrays of numbers
(e.g. coordinates) but costs arrays of other values only one test, not one
per element.
@param[in] len the number of elements
@return whether the SAX parser accepted the array and the values read
*/
bool enter_cbor_array(const std::size_t len)
{
return enter_array(len) && (len == 0 || parse_cbor_array_start(std::integral_constant<bool, bulk_scan> {}));
}
/// read the scalars the array just opened begins with; out of line, so
/// that @ref parse_cbor_value stays small enough to be inlined into
/// @ref parse_cbor_internal
JSON_HEDLEY_NEVER_INLINE
bool parse_cbor_array_start(std::true_type /*bulk*/)
{
return parse_cbor_scalar_run(container_stack.back().remaining);
}
/// input that is not contiguous: every value is read by parse_cbor_value
bool parse_cbor_array_start(std::false_type /*bulk*/) const noexcept
{
return true;
}
/*!
@brief read a CBOR value and everything nested inside it
@@ -19634,7 +19481,6 @@ class binary_reader
// pop_back() below, which destroys the container_stack element
// it would otherwise alias
const container_frame top = container_stack.back();
const bool is_object = top.is_object;
bool at_end = false;
if (top.remaining != npos)
@@ -19646,7 +19492,7 @@ class binary_reader
{
// claim the element about to be read
--container_stack.back().remaining;
if (is_object)
if (top.is_object)
{
get();
}
@@ -19659,7 +19505,7 @@ class binary_reader
// Testing for it consumes a byte, which is the first byte
// of the next element when it is not one.
at_end = (get() == 0xFF);
fetch = is_object;
fetch = top.is_object;
}
if (at_end)
@@ -19676,7 +19522,7 @@ class binary_reader
continue;
}
if (is_object)
if (top.is_object)
{
if (JSON_HEDLEY_UNLIKELY(!skip_cbor_key_tags(tag_handler)))
{
@@ -21621,13 +21467,13 @@ class binary_reader
return sax->null();
case 0xFB:
return sax->number_float(static_cast<number_float_t>(-1.0), no_source);
return sax->number_float(static_cast<number_float_t>(-1.0), "");
case 0xFC:
return sax->number_float(static_cast<number_float_t>(0.0), no_source);
return sax->number_float(static_cast<number_float_t>(0.0), "");
case 0xFD:
return sax->number_float(static_cast<number_float_t>(1.0), no_source);
return sax->number_float(static_cast<number_float_t>(1.0), "");
case 0xFF: // empty string
{
@@ -22131,7 +21977,7 @@ class binary_reader
{
return out_of_range_error(exception_id::number_overflow, "number overflow", "value");
}
return sax->number_float(result, no_source);
return sax->number_float(result, "");
}
/*!
@@ -22191,7 +22037,7 @@ class binary_reader
}();
return sax->number_float((half & 0x8000u) != 0
? static_cast<number_float_t>(-val)
: static_cast<number_float_t>(val), no_source);
: static_cast<number_float_t>(val), "");
}
/*!
@@ -22516,10 +22362,6 @@ class binary_reader
/// the SAX parser
json_sax_t* sax = nullptr;
/// the (empty) source text of every number_float event: binary formats
/// have none, and a string_t built for each float would cost more
const string_t no_source{};
/// the containers that have been opened and not closed yet; see @ref container_frame
std::vector<container_frame> container_stack{};
-74
View File
@@ -29,9 +29,7 @@ using nlohmann::json;
#include "round_trip_corpus.hpp"
#include "test_utils.hpp"
#include "sax_countdown.hpp"
#include "sax_event_loggers.hpp"
using utils::SaxCountdown;
using utils::SaxEventLogger;
TEST_CASE("CBOR")
@@ -2448,78 +2446,6 @@ TEST_CASE("CBOR SAX parsing stops at every event")
CHECK(count_events(std::vector<std::uint8_t>({0xBF, 0x61, 'a', 0x9F, 0x01, 0xFF, 0xFF})) == 6);
}
TEST_CASE("CBOR arrays of scalars from contiguous and non-contiguous input")
{
// contiguous input reads the integers, floats, booleans, and nulls an
// array begins with in one loop, other input value by value; both must
// give the same events, value, or error for the complete document and for
// every truncation of it, and stop at every event a SAX handler rejects
const std::vector<std::uint8_t> cbor =
{
0xA3, // map(3)
0x61, 'a', // "a"
0x98, 0x16, // array(22)
0x00, 0x17, 0x18, 0xFF, 0x19, 0x01, 0x00, // 0, 23, 255, 256
0x1A, 0x00, 0x01, 0x00, 0x00, // 65536
0x1B, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 18446744073709551615
0x20, 0x37, 0x38, 0xFF, 0x39, 0x01, 0x00, // -1, -24, -256, -257
0x3A, 0x00, 0x01, 0x00, 0x00, // -65537
0x3B, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // -18446744073709551616 (a float)
0x3B, 0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // -9223372036854775808
0xF4, 0xF5, 0xF6, // false, true, null
0xFA, 0x3F, 0xC0, 0x00, 0x00, // 1.5 (float32)
0xFB, 0xBF, 0xB9, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9A, // -0.1
0xF9, 0x3C, 0x00, // 1.0 (float16, ends the loop)
0x01, 0x61, 'a', // 1, "a"
0xFB, 0x40, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 2.5
0x61, 'b', // "b"
0x83, // array(3)
0x82, 0xFB, 0x3F, 0xF4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // [1.25,
0xFB, 0xC0, 0x0E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // -3.75]
0x82, 0xFB, 0x3F, 0xF4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // [1.25,
0xFB, 0xC0, 0x0E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // -3.75]
0x80, // []
0x61, 'c', // "c"
0x84, 0xA0, 0x01, 0xC1, 0x02, 0x81, 0x03 // [{}, 1, tag(1) 2, [3]]
};
const auto tags = json::cbor_tag_handler_t::ignore;
const json j = json::from_cbor(cbor, true, true, tags);
CHECK(j["a"].size() == 22);
CHECK(j["a"][5] == 18446744073709551615u);
CHECK(j["a"][11].is_number_float());
CHECK(j["a"][11] == -18446744073709551616.0);
CHECK(j["a"][12] == (std::numeric_limits<std::int64_t>::min)());
CHECK(j["a"][17] == -0.1);
CHECK(j["b"] == json::parse("[[1.25, -3.75], [1.25, -3.75], []]"));
CHECK(j["c"] == json::parse("[{}, 1, 2, [3]]"));
for (std::size_t length = 0; length <= cbor.size(); ++length)
{
CAPTURE(length)
const std::vector<std::uint8_t> contiguous(cbor.begin(), cbor.begin() + static_cast<std::ptrdiff_t>(length));
const std::list<std::uint8_t> bytewise(contiguous.begin(), contiguous.end());
CHECK(json::from_cbor(contiguous, true, false, tags).dump() == json::from_cbor(bytewise.begin(), bytewise.end(), true, false, tags).dump());
SaxEventLogger from_contiguous;
SaxEventLogger from_bytewise;
CHECK(json::sax_parse(contiguous, &from_contiguous, json::input_format_t::cbor, true, false, false, tags)
== json::sax_parse(bytewise.begin(), bytewise.end(), &from_bytewise, json::input_format_t::cbor, true, false, false, tags));
CHECK(from_contiguous.events == from_bytewise.events);
}
for (int events = 0; events <= 60; ++events)
{
CAPTURE(events)
const std::list<std::uint8_t> bytewise(cbor.begin(), cbor.end());
SaxCountdown from_contiguous(events);
SaxCountdown from_bytewise(events);
CHECK(json::sax_parse(cbor, &from_contiguous, json::input_format_t::cbor, true, false, false, tags)
== json::sax_parse(bytewise.begin(), bytewise.end(), &from_bytewise, json::input_format_t::cbor, true, false, false, tags));
}
}
TEST_CASE("CBOR indefinite-length strings do not recurse per chunk")
{
// Reading an indefinite-length string or byte array used to call itself
+24
View File
@@ -443,6 +443,30 @@ TEST_CASE("json_view from_json")
CHECK(d.root().materialize().get<point>().x == 3);
}
SECTION("escaped keys: members are found by their decoded names")
{
// keys are decoded before they are compared, as by parse(); a lookup
// that compares the raw bytes of the source would miss these members
const json_document d = json_document::parse(R"([{"\u0078": 1, "y": 2}, {"V\u0061lue": 3, "Te\u0078t": "t"}, {"\u0062": "escaped"}, {"x": 1, "\u0078": 3, "y": 2}])");
const json j = d.root().materialize();
CHECK(d.root()[0].get<point>().x == 1);
CHECK(d.root()[0].get<point>().y == 2);
CHECK((d.root()[0].get<point>() == j[0].get<point>()));
CHECK(d.root()[1].get<with_names>().value == 3);
CHECK(d.root()[1].get<with_names>().text == "t");
CHECK(d.root()[2].get<with_defaults>().b == "escaped");
CHECK(d.root()[2].get<with_defaults>().a == 1);
// a duplicate key written once plain and once escaped: still the first member
CHECK(d.root()[3].get<point>().x == 1);
CHECK(j[3].get<point>().x == 3);
#if !defined(JSON_NOEXCEPTION) && !JSON_DIAGNOSTICS && !JSON_DIAGNOSTIC_POSITIONS
// an escaped backslash is part of the decoded key: "x\\" is not "x"
const json_document e = json_document::parse(R"({"x\\": 1, "y": 2})");
CHECK(exception_of_call([&] { static_cast<void>(e.root().get<point>()); }) == exception_of_call([&] { static_cast<void>(e.root().materialize().get<point>()); }));
CHECK(exception_of_call([&] { static_cast<void>(e.root().get<point>()); }) == "[json.exception.out_of_range.403] key 'x' not found");
#endif
}
SECTION("editable documents and ordered_json")
{
json_editable_document e = json_editable_document::parse(R"({"x": 1, "y": 2})");