mirror of
https://github.com/nlohmann/json.git
synced 2026-10-03 21:20:30 +00:00
Add an error_handler parameter for UTF-8 to the binary readers and writers
Adds error_handler_t::keep (invalid UTF-8 sequences are left unchanged) as a fourth error_handler_t value, and threads an error_handler parameter through the binary writers and readers: - to_cbor/to_ubjson/to_bjdata/to_bson gain a trailing error_handler parameter (default strict, matching their existing type_error.316 behavior); keep writes a string value or object key's bytes as is instead of throwing, and replace/ignore sanitize it exactly like dump() would, including for the BSON length prefix. to_msgpack and to_bon8 are unchanged. - from_cbor/from_msgpack/from_ubjson/from_bjdata/from_bson gain a trailing error_handler parameter (default keep, i.e. the lenient behavior every binary reader had in 3.12.0 and still has after #5741); strict checks every string value and object key and raises parse_error.113 for ill-formed UTF-8, honoring allow_exceptions; replace/ignore sanitize it like dump() would. from_bon8 is unchanged, since UTF-8 lead bytes are structural there. dump()'s own keep support writes ill-formed bytes as is, even with ensure_ascii, while still \u-escaping well-formed characters around them as usual. The UTF-8 validity check (is_valid_utf8) and the replace/ignore sanitizing logic (sanitize_utf8) now live in string_utils.hpp, shared by the serializer and the binary reader/writer; error_handler_t itself moved to its own header (detail/output/error_handler.hpp) so that string_utils.hpp does not need to depend on serializer.hpp. See #5529 and #5741. Signed-off-by: Niels Lohmann <mail@nlohmann.me>
This commit is contained in:
Niels Lohmann
@@ -0,0 +1,346 @@
|
||||
// __ _____ _____ _____
|
||||
// __| | __| | | | JSON for Modern C++ (supporting code)
|
||||
// | | |__ | | | | | | version 3.12.0
|
||||
// |_____|_____|_____|_|___| https://github.com/nlohmann/json
|
||||
//
|
||||
// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
#include "doctest_compatibility.h"
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
using nlohmann::json;
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
struct ill_formed_case
|
||||
{
|
||||
const char* name;
|
||||
std::string bytes;
|
||||
};
|
||||
|
||||
// RFC 3629 ill-formed sequences used throughout this file, plus one
|
||||
// well-formed sequence for contrast
|
||||
const std::vector<ill_formed_case> ill_formed_cases =
|
||||
{
|
||||
{"overlong", "\xC0\xAE"},
|
||||
{"lone_0xFF", "\xFF"},
|
||||
{"truncated", "\xE2\x82"},
|
||||
{"surrogate", "\xED\xA0\x80"},
|
||||
};
|
||||
|
||||
const std::string valid_sequence = "\xC3\xA9"; // U+00E9, "é"
|
||||
|
||||
using eh = json::error_handler_t;
|
||||
const std::vector<eh> all_handlers = {eh::strict, eh::replace, eh::ignore, eh::keep};
|
||||
|
||||
// what dump()+parse() produces for a sanitizing error_handler; this is the
|
||||
// ground truth every binary writer/reader is checked against
|
||||
std::string dump_and_parse(const std::string& raw, eh error_handler)
|
||||
{
|
||||
return json::parse(json(raw).dump(-1, ' ', false, error_handler)).get<std::string>();
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
TEST_CASE("UTF-8 error_handler for the binary readers and writers")
|
||||
{
|
||||
SECTION("writers: string value")
|
||||
{
|
||||
for (const auto& c : ill_formed_cases)
|
||||
{
|
||||
CAPTURE(c.name);
|
||||
const json jval = c.bytes;
|
||||
|
||||
CHECK_THROWS_AS(json::to_cbor(jval, eh::strict), json::type_error&);
|
||||
CHECK_THROWS_AS(json::to_ubjson(jval, false, false, eh::strict), json::type_error&);
|
||||
CHECK_THROWS_AS(json::to_bjdata(jval, false, false, json::bjdata_version_t::draft2, eh::strict), json::type_error&);
|
||||
{
|
||||
json jobj;
|
||||
jobj["k"] = jval;
|
||||
CHECK_THROWS_AS(json::to_bson(jobj, eh::strict), json::type_error&);
|
||||
}
|
||||
|
||||
for (const auto h :
|
||||
{
|
||||
eh::replace, eh::ignore
|
||||
})
|
||||
{
|
||||
CAPTURE(static_cast<int>(h));
|
||||
const std::string expected = dump_and_parse(c.bytes, h);
|
||||
|
||||
CHECK(json::from_cbor(json::to_cbor(jval, h)).get<std::string>() == expected);
|
||||
CHECK(json::from_ubjson(json::to_ubjson(jval, false, false, h)).get<std::string>() == expected);
|
||||
CHECK(json::from_bjdata(json::to_bjdata(jval, false, false, json::bjdata_version_t::draft2, h)).get<std::string>() == expected);
|
||||
{
|
||||
json jobj;
|
||||
jobj["k"] = jval;
|
||||
const auto bytes = json::to_bson(jobj, h);
|
||||
CHECK(json::from_bson(bytes)["k"].get<std::string>() == expected);
|
||||
}
|
||||
}
|
||||
|
||||
// keep: the writer passes the ill-formed bytes through unchanged,
|
||||
// exactly as every binary writer did before this parameter existed
|
||||
CHECK(json::from_cbor(json::to_cbor(jval, eh::keep)).get<std::string>() == c.bytes);
|
||||
CHECK(json::from_ubjson(json::to_ubjson(jval, false, false, eh::keep)).get<std::string>() == c.bytes);
|
||||
CHECK(json::from_bjdata(json::to_bjdata(jval, false, false, json::bjdata_version_t::draft2, eh::keep)).get<std::string>() == c.bytes);
|
||||
{
|
||||
json jobj;
|
||||
jobj["k"] = jval;
|
||||
const auto bytes = json::to_bson(jobj, eh::keep);
|
||||
CHECK(json::from_bson(bytes)["k"].get<std::string>() == c.bytes);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
SECTION("writers: object key")
|
||||
{
|
||||
for (const auto& c : ill_formed_cases)
|
||||
{
|
||||
CAPTURE(c.name);
|
||||
json jobj;
|
||||
jobj[c.bytes] = 1;
|
||||
|
||||
CHECK_THROWS_AS(json::to_cbor(jobj, eh::strict), json::type_error&);
|
||||
CHECK_THROWS_AS(json::to_ubjson(jobj, false, false, eh::strict), json::type_error&);
|
||||
CHECK_THROWS_AS(json::to_bjdata(jobj, false, false, json::bjdata_version_t::draft2, eh::strict), json::type_error&);
|
||||
CHECK_THROWS_AS(json::to_bson(jobj, eh::strict), json::type_error&);
|
||||
|
||||
for (const auto h :
|
||||
{
|
||||
eh::replace, eh::ignore
|
||||
})
|
||||
{
|
||||
CAPTURE(static_cast<int>(h));
|
||||
const std::string expected = dump_and_parse(c.bytes, h);
|
||||
|
||||
CHECK(json::from_cbor(json::to_cbor(jobj, h)).begin().key() == expected);
|
||||
CHECK(json::from_ubjson(json::to_ubjson(jobj, false, false, h)).begin().key() == expected);
|
||||
CHECK(json::from_bjdata(json::to_bjdata(jobj, false, false, json::bjdata_version_t::draft2, h)).begin().key() == expected);
|
||||
CHECK(json::from_bson(json::to_bson(jobj, h)).begin().key() == expected);
|
||||
}
|
||||
|
||||
// keep: object keys round-trip unchanged too
|
||||
CHECK(json::from_cbor(json::to_cbor(jobj, eh::keep)).begin().key() == c.bytes);
|
||||
CHECK(json::from_ubjson(json::to_ubjson(jobj, false, false, eh::keep)).begin().key() == c.bytes);
|
||||
CHECK(json::from_bjdata(json::to_bjdata(jobj, false, false, json::bjdata_version_t::draft2, eh::keep)).begin().key() == c.bytes);
|
||||
CHECK(json::from_bson(json::to_bson(jobj, eh::keep)).begin().key() == c.bytes);
|
||||
}
|
||||
}
|
||||
|
||||
SECTION("readers: string value")
|
||||
{
|
||||
for (const auto& c : ill_formed_cases)
|
||||
{
|
||||
CAPTURE(c.name);
|
||||
|
||||
// bytes produced the lenient (keep) way, as any binary reader
|
||||
// accepted them before this parameter existed
|
||||
const auto cbor_bytes = json::to_cbor(json(c.bytes), eh::keep);
|
||||
const auto msgpack_bytes = json::to_msgpack(json(c.bytes)); // to_msgpack has no error_handler; always pass-through
|
||||
const auto ubjson_bytes = json::to_ubjson(json(c.bytes), false, false, eh::keep);
|
||||
const auto bjdata_bytes = json::to_bjdata(json(c.bytes), false, false, json::bjdata_version_t::draft2, eh::keep);
|
||||
const auto bson_bytes = [&c]
|
||||
{
|
||||
json jobj;
|
||||
jobj["k"] = c.bytes;
|
||||
return json::to_bson(jobj, eh::keep);
|
||||
}();
|
||||
|
||||
// keep (the default): bytes are kept unchanged
|
||||
CHECK(json::from_cbor(cbor_bytes).get<std::string>() == c.bytes);
|
||||
CHECK(json::from_msgpack(msgpack_bytes).get<std::string>() == c.bytes);
|
||||
CHECK(json::from_ubjson(ubjson_bytes).get<std::string>() == c.bytes);
|
||||
CHECK(json::from_bjdata(bjdata_bytes).get<std::string>() == c.bytes);
|
||||
CHECK(json::from_bson(bson_bytes)["k"].get<std::string>() == c.bytes);
|
||||
|
||||
// strict: parse_error.113, discarded (not thrown) when allow_exceptions is false
|
||||
CHECK_THROWS_AS(json::from_cbor(cbor_bytes, true, true, json::cbor_tag_handler_t::error, eh::strict), json::parse_error&);
|
||||
CHECK(json::from_cbor(cbor_bytes, true, false, json::cbor_tag_handler_t::error, eh::strict).is_discarded());
|
||||
CHECK_THROWS_AS(json::from_msgpack(msgpack_bytes, true, true, eh::strict), json::parse_error&);
|
||||
CHECK(json::from_msgpack(msgpack_bytes, true, false, eh::strict).is_discarded());
|
||||
CHECK_THROWS_AS(json::from_ubjson(ubjson_bytes, true, true, eh::strict), json::parse_error&);
|
||||
CHECK(json::from_ubjson(ubjson_bytes, true, false, eh::strict).is_discarded());
|
||||
CHECK_THROWS_AS(json::from_bjdata(bjdata_bytes, true, true, eh::strict), json::parse_error&);
|
||||
CHECK(json::from_bjdata(bjdata_bytes, true, false, eh::strict).is_discarded());
|
||||
CHECK_THROWS_AS(json::from_bson(bson_bytes, true, true, eh::strict), json::parse_error&);
|
||||
CHECK(json::from_bson(bson_bytes, true, false, eh::strict).is_discarded());
|
||||
|
||||
// replace / ignore: match what dump() would have sanitized the same bytes to
|
||||
for (const auto h :
|
||||
{
|
||||
eh::replace, eh::ignore
|
||||
})
|
||||
{
|
||||
CAPTURE(static_cast<int>(h));
|
||||
const std::string expected = dump_and_parse(c.bytes, h);
|
||||
|
||||
CHECK(json::from_cbor(cbor_bytes, true, true, json::cbor_tag_handler_t::error, h).get<std::string>() == expected);
|
||||
CHECK(json::from_msgpack(msgpack_bytes, true, true, h).get<std::string>() == expected);
|
||||
CHECK(json::from_ubjson(ubjson_bytes, true, true, h).get<std::string>() == expected);
|
||||
CHECK(json::from_bjdata(bjdata_bytes, true, true, h).get<std::string>() == expected);
|
||||
CHECK(json::from_bson(bson_bytes, true, true, h)["k"].get<std::string>() == expected);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
SECTION("readers: object key")
|
||||
{
|
||||
for (const auto& c : ill_formed_cases)
|
||||
{
|
||||
CAPTURE(c.name);
|
||||
|
||||
json jobj;
|
||||
jobj[c.bytes] = 1;
|
||||
const auto cbor_bytes = json::to_cbor(jobj, eh::keep);
|
||||
const auto msgpack_bytes = json::to_msgpack(jobj);
|
||||
const auto ubjson_bytes = json::to_ubjson(jobj, false, false, eh::keep);
|
||||
const auto bjdata_bytes = json::to_bjdata(jobj, false, false, json::bjdata_version_t::draft2, eh::keep);
|
||||
const auto bson_bytes = json::to_bson(jobj, eh::keep);
|
||||
|
||||
CHECK(json::from_cbor(cbor_bytes).begin().key() == c.bytes);
|
||||
CHECK(json::from_msgpack(msgpack_bytes).begin().key() == c.bytes);
|
||||
CHECK(json::from_ubjson(ubjson_bytes).begin().key() == c.bytes);
|
||||
CHECK(json::from_bjdata(bjdata_bytes).begin().key() == c.bytes);
|
||||
CHECK(json::from_bson(bson_bytes).begin().key() == c.bytes);
|
||||
|
||||
CHECK_THROWS_AS(json::from_cbor(cbor_bytes, true, true, json::cbor_tag_handler_t::error, eh::strict), json::parse_error&);
|
||||
CHECK_THROWS_AS(json::from_msgpack(msgpack_bytes, true, true, eh::strict), json::parse_error&);
|
||||
CHECK_THROWS_AS(json::from_ubjson(ubjson_bytes, true, true, eh::strict), json::parse_error&);
|
||||
CHECK_THROWS_AS(json::from_bjdata(bjdata_bytes, true, true, eh::strict), json::parse_error&);
|
||||
CHECK_THROWS_AS(json::from_bson(bson_bytes, true, true, eh::strict), json::parse_error&);
|
||||
|
||||
for (const auto h :
|
||||
{
|
||||
eh::replace, eh::ignore
|
||||
})
|
||||
{
|
||||
CAPTURE(static_cast<int>(h));
|
||||
const std::string expected = dump_and_parse(c.bytes, h);
|
||||
|
||||
CHECK(json::from_cbor(cbor_bytes, true, true, json::cbor_tag_handler_t::error, h).begin().key() == expected);
|
||||
CHECK(json::from_msgpack(msgpack_bytes, true, true, h).begin().key() == expected);
|
||||
CHECK(json::from_ubjson(ubjson_bytes, true, true, h).begin().key() == expected);
|
||||
CHECK(json::from_bjdata(bjdata_bytes, true, true, h).begin().key() == expected);
|
||||
CHECK(json::from_bson(bson_bytes, true, true, h).begin().key() == expected);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
SECTION("well-formed UTF-8 is unaffected by error_handler")
|
||||
{
|
||||
const json jval = valid_sequence;
|
||||
json jobj;
|
||||
jobj[valid_sequence] = valid_sequence;
|
||||
|
||||
for (const auto h : all_handlers)
|
||||
{
|
||||
CAPTURE(static_cast<int>(h));
|
||||
|
||||
CHECK(json::from_cbor(json::to_cbor(jval, h)).get<std::string>() == valid_sequence);
|
||||
CHECK(json::from_ubjson(json::to_ubjson(jval, false, false, h)).get<std::string>() == valid_sequence);
|
||||
CHECK(json::from_bjdata(json::to_bjdata(jval, false, false, json::bjdata_version_t::draft2, h)).get<std::string>() == valid_sequence);
|
||||
CHECK(json::from_bson(json::to_bson(jobj, h)).begin().key() == valid_sequence);
|
||||
|
||||
CHECK(json::from_cbor(json::to_cbor(jval, eh::keep), true, true, json::cbor_tag_handler_t::error, h).get<std::string>() == valid_sequence);
|
||||
CHECK(json::from_msgpack(json::to_msgpack(jval), true, true, h).get<std::string>() == valid_sequence);
|
||||
}
|
||||
}
|
||||
|
||||
SECTION("dump() with error_handler_t::keep writes raw bytes as is")
|
||||
{
|
||||
for (const auto& c : ill_formed_cases)
|
||||
{
|
||||
CAPTURE(c.name);
|
||||
|
||||
const json jval = c.bytes;
|
||||
const std::string dumped = jval.dump(-1, ' ', false, eh::keep);
|
||||
CHECK(dumped.find(c.bytes) != std::string::npos);
|
||||
|
||||
// even with ensure_ascii, the ill-formed bytes are written as is
|
||||
const std::string dumped_ascii = jval.dump(-1, ' ', true, eh::keep);
|
||||
CHECK(dumped_ascii.find(c.bytes) != std::string::npos);
|
||||
}
|
||||
|
||||
// well-formed characters around an ill-formed sequence are still
|
||||
// escaped as usual under ensure_ascii
|
||||
const json mixed = valid_sequence + ill_formed_cases[1].bytes; // "é" + lone 0xFF
|
||||
const std::string dumped_mixed = mixed.dump(-1, ' ', true, eh::keep);
|
||||
CHECK(dumped_mixed.find("\\u00e9") != std::string::npos);
|
||||
CHECK(dumped_mixed.find(ill_formed_cases[1].bytes) != std::string::npos);
|
||||
|
||||
// the byte that ends an ill-formed sequence is read again, so a quote,
|
||||
// a backslash, or a control character after it is still escaped, and
|
||||
// a well-formed code point after it is escaped under ensure_ascii
|
||||
for (const bool ensure_ascii :
|
||||
{
|
||||
false, true
|
||||
})
|
||||
{
|
||||
CAPTURE(ensure_ascii);
|
||||
CHECK(json("\xC3\"").dump(-1, ' ', ensure_ascii, eh::keep) == "\"\xC3\\\"\"");
|
||||
CHECK(json("\xC3\\").dump(-1, ' ', ensure_ascii, eh::keep) == "\"\xC3\\\\\"");
|
||||
CHECK(json("\xC3\n").dump(-1, ' ', ensure_ascii, eh::keep) == "\"\xC3\\n\"");
|
||||
CHECK(json("\xE2\x82\"").dump(-1, ' ', ensure_ascii, eh::keep) == "\"\xE2\x82\\\"\"");
|
||||
CHECK(json("\xFF\"").dump(-1, ' ', ensure_ascii, eh::keep) == "\"\xFF\\\"\"");
|
||||
CHECK(json("a\xE2\x82").dump(-1, ' ', ensure_ascii, eh::keep) == "\"a\xE2\x82\"");
|
||||
}
|
||||
CHECK(json("\xC3\xC3\xA9").dump(-1, ' ', false, eh::keep) == "\"\xC3\xC3\xA9\"");
|
||||
CHECK(json("\xC3\xC3\xA9").dump(-1, ' ', true, eh::keep) == "\"\xC3\\u00e9\"");
|
||||
}
|
||||
|
||||
SECTION("to_msgpack and to_bon8 are not affected by error_handler")
|
||||
{
|
||||
const json jval = ill_formed_cases[1].bytes; // lone 0xFF
|
||||
|
||||
// to_msgpack has no error_handler parameter; the bytes are always
|
||||
// passed through, as MessagePack's spec allows
|
||||
CHECK(json::from_msgpack(json::to_msgpack(jval)).get<std::string>() == ill_formed_cases[1].bytes);
|
||||
|
||||
// to_bon8 has no error_handler parameter either; UTF-8 is structural
|
||||
// for BON8, so it always rejects ill-formed input
|
||||
CHECK_THROWS_AS(json::to_bon8(jval), json::type_error&);
|
||||
}
|
||||
|
||||
SECTION("allow_exceptions=false with error_handler_t::strict discards the value")
|
||||
{
|
||||
const auto bytes = json::to_cbor(json(ill_formed_cases[0].bytes), eh::keep);
|
||||
const json result = json::from_cbor(bytes, true, false, json::cbor_tag_handler_t::error, eh::strict);
|
||||
CHECK(result.is_discarded());
|
||||
}
|
||||
|
||||
SECTION("default parameters are unchanged")
|
||||
{
|
||||
const json jval = ill_formed_cases[0].bytes;
|
||||
|
||||
// to_*: the default error_handler is strict, so ill-formed input still throws
|
||||
CHECK_THROWS_AS(json::to_cbor(jval), json::type_error&);
|
||||
CHECK_THROWS_AS(json::to_ubjson(jval), json::type_error&);
|
||||
CHECK_THROWS_AS(json::to_bjdata(jval), json::type_error&);
|
||||
{
|
||||
json jobj;
|
||||
jobj["k"] = jval;
|
||||
CHECK_THROWS_AS(json::to_bson(jobj), json::type_error&);
|
||||
}
|
||||
|
||||
// from_*: the default error_handler is keep, so ill-formed bytes are
|
||||
// still accepted unchanged, exactly as in release 3.12.0
|
||||
const auto cbor_bytes = json::to_cbor(jval, eh::keep);
|
||||
CHECK(json::from_cbor(cbor_bytes).get<std::string>() == ill_formed_cases[0].bytes);
|
||||
const auto ubjson_bytes = json::to_ubjson(jval, false, false, eh::keep);
|
||||
CHECK(json::from_ubjson(ubjson_bytes).get<std::string>() == ill_formed_cases[0].bytes);
|
||||
const auto bjdata_bytes = json::to_bjdata(jval, false, false, json::bjdata_version_t::draft2, eh::keep);
|
||||
CHECK(json::from_bjdata(bjdata_bytes).get<std::string>() == ill_formed_cases[0].bytes);
|
||||
const auto msgpack_bytes = json::to_msgpack(jval);
|
||||
CHECK(json::from_msgpack(msgpack_bytes).get<std::string>() == ill_formed_cases[0].bytes);
|
||||
json bson_obj;
|
||||
bson_obj["k"] = jval;
|
||||
const auto bson_bytes = json::to_bson(bson_obj, eh::keep);
|
||||
CHECK(json::from_bson(bson_bytes)["k"].get<std::string>() == ill_formed_cases[0].bytes);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user