amorfo77/json
1
0
Fork 0
mirror of https://github.com/nlohmann/json.git synced 2026-10-03 21:20:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add error_handler_t::keep to copy invalid UTF-8 bytes unchanged

Browse Source 
error_handler_t::ignore drops invalid bytes although its docs promised to copy them (#4552). Add keep, which copies each ill-formed subsequence byte-for-byte while still escaping valid characters.

Fixes #4552

Signed-off-by: Niels Lohmann <mail@nlohmann.me>
This commit is contained in:
Niels Lohmann
2026-09-26 07:37:05 +02:00
parent f422b753cc
commit 8aabb9981f
15 changed files with 283 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/src/unit-regression2.cpp
+9
View File
@@ -765,6 +765,15 @@ TEST_CASE("regression tests 2")
CHECK(j == k);
}
SECTION("issue #4552 - UTF-8 invalid characters are not always ignored when dumping with error_handler_t::ignore")
{
json node;
node["test"] = "test\334\005";
CHECK(node.dump(-1, ' ', false, json::error_handler_t::ignore) == "{\"test\":\"test\\u0005\"}");
CHECK(node.dump(-1, ' ', false, json::error_handler_t::keep) == "{\"test\":\"test\334\\u0005\"}");
CHECK(node.dump(-1, ' ', true, json::error_handler_t::keep) == "{\"test\":\"test\334\\u0005\"}");
}
}
TEST_CASE("regression test - parser callback must not lose a duplicate key's prior value")
tests/src/unit-serialization.cpp
+37
View File
@@ -92,6 +92,8 @@ TEST_CASE("serialization")
CHECK(j.dump(-1, ' ', false, json::error_handler_t::ignore) == "\"äü\"");
CHECK(j.dump(-1, ' ', false, json::error_handler_t::replace) == "\"ä\xEF\xBF\xBDü\"");
CHECK(j.dump(-1, ' ', true, json::error_handler_t::replace) == "\"\\u00e4\\ufffd\\u00fc\"");
CHECK(j.dump(-1, ' ', false, json::error_handler_t::keep) == "\"ä\xA9ü\"");
CHECK(j.dump(-1, ' ', true, json::error_handler_t::keep) == "\"\\u00e4\xA9\\u00fc\"");
}
SECTION("invalid character (regression guard for shared UTF-8 decoder, see #5529)")
@@ -114,6 +116,8 @@ TEST_CASE("serialization")
CHECK(j.dump(-1, ' ', false, json::error_handler_t::ignore) == "\"123\"");
CHECK(j.dump(-1, ' ', false, json::error_handler_t::replace) == "\"123\xEF\xBF\xBD\"");
CHECK(j.dump(-1, ' ', true, json::error_handler_t::replace) == "\"123\\ufffd\"");
CHECK(j.dump(-1, ' ', false, json::error_handler_t::keep) == "\"123\xC2\"");
CHECK(j.dump(-1, ' ', true, json::error_handler_t::keep) == "\"123\xC2\"");
}
SECTION("unexpected character")
@@ -126,6 +130,39 @@ TEST_CASE("serialization")
CHECK(j.dump(-1, ' ', false, json::error_handler_t::ignore) == "\"123456\"");
CHECK(j.dump(-1, ' ', false, json::error_handler_t::replace) == "\"123\xEF\xBF\xBD\x34\x35\x36\"");
CHECK(j.dump(-1, ' ', true, json::error_handler_t::replace) == "\"123\\ufffd456\"");
CHECK(j.dump(-1, ' ', false, json::error_handler_t::keep) == "\"123\xF1\xB0\x34\x35\x36\"");
CHECK(j.dump(-1, ' ', true, json::error_handler_t::keep) == "\"123\xF1\xB0\x34\x35\x36\"");
}
SECTION("keep: valid characters are still escaped")
{
// an invalid byte followed by characters that must be escaped
const json j = "\xC2\"\\\n\xFF\x05";
CHECK(j.dump(-1, ' ', false, json::error_handler_t::keep) == "\"\xC2\\\"\\\\\\n\xFF\\u0005\"");
CHECK(j.dump(-1, ' ', true, json::error_handler_t::keep) == "\"\xC2\\\"\\\\\\n\xFF\\u0005\"");
}
SECTION("keep: truncated multibyte sequences")
{
CHECK(json("\xF0\x9F\x98").dump(-1, ' ', false, json::error_handler_t::keep) == "\"\xF0\x9F\x98\"");
CHECK(json("\xF0\x9F\x98").dump(-1, ' ', true, json::error_handler_t::keep) == "\"\xF0\x9F\x98\"");
CHECK(json("\xF0\x9F\x98" "a").dump(-1, ' ', false, json::error_handler_t::keep) == "\"\xF0\x9F\x98" "a\"");
CHECK(json("\xF0\x9F\x98" "a").dump(-1, ' ', true, json::error_handler_t::keep) == "\"\xF0\x9F\x98" "a\"");
}
SECTION("keep: long string with many invalid bytes")
{
// exceeds the internal string buffer several times
std::string input;
std::string expected = "\"";
for (int i = 0; i < 2000; ++i)
{
input += "\xFF\xE2\x82\n\xC3\xA4";
expected += "\xFF\xE2\x82\\n\xC3\xA4";
}
expected += "\"";
const json j = input;
CHECK(j.dump(-1, ' ', false, json::error_handler_t::keep) == expected);
}
SECTION("U+FFFD Substitution of Maximal Subparts")
tests/src/unit-unicode2.cpp
+25 -1
View File
@@ -14,6 +14,7 @@
#include <nlohmann/json.hpp>
using nlohmann::json;
#include <algorithm>
#include <fstream>
#include <sstream>
#include <iostream>
@@ -75,8 +76,11 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
static std::string s_replaced2;
static std::string s_replaced_ascii;
static std::string s_replaced2_ascii;
static std::string s_kept;
static std::string s_kept2;
static std::string s_kept_ascii;
// dumping with ignore/replace must not throw in any case
// dumping with ignore/replace/keep must not throw in any case
s_ignored = j.dump(-1, ' ', false, json::error_handler_t::ignore);
s_ignored2 = j2.dump(-1, ' ', false, json::error_handler_t::ignore);
s_ignored_ascii = j.dump(-1, ' ', true, json::error_handler_t::ignore);
@@ -85,6 +89,9 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
s_replaced2 = j2.dump(-1, ' ', false, json::error_handler_t::replace);
s_replaced_ascii = j.dump(-1, ' ', true, json::error_handler_t::replace);
s_replaced2_ascii = j2.dump(-1, ' ', true, json::error_handler_t::replace);
s_kept = j.dump(-1, ' ', false, json::error_handler_t::keep);
s_kept2 = j2.dump(-1, ' ', false, json::error_handler_t::keep);
s_kept_ascii = j.dump(-1, ' ', true, json::error_handler_t::keep);
if (success_expected)
{
@@ -94,6 +101,7 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
// all dumps should agree on the string
CHECK(s_strict == s_ignored);
CHECK(s_strict == s_replaced);
CHECK(s_strict == s_kept);
}
else
{
@@ -105,6 +113,20 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
// check that replace string contains a replacement character
CHECK(s_replaced.find("\xEF\xBF\xBD") != std::string::npos);
// ignore drops the invalid bytes, keep copies them
CHECK(s_ignored != s_kept);
CHECK(s_ignored_ascii != s_kept_ascii);
// unless a byte needs escaping, keep copies the input unchanged
const bool needs_escaping = std::any_of(json_string.begin(), json_string.end(), [](char c)
{
return static_cast<unsigned char>(c) < 0x20 || c == '"' || c == '\\';
});
if (!needs_escaping)
{
CHECK(s_kept == "\"" + json_string + "\"");
}
}
// check that prefix and suffix are preserved
@@ -116,6 +138,8 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
CHECK(s_replaced2.substr(s_replaced2.size() - 4, 3) == "xyz");
CHECK(s_replaced2_ascii.substr(1, 3) == "abc");
CHECK(s_replaced2_ascii.substr(s_replaced2_ascii.size() - 4, 3) == "xyz");
CHECK(s_kept2.substr(1, 3) == "abc");
CHECK(s_kept2.substr(s_kept2.size() - 4, 3) == "xyz");
}
void check_utf8string(bool success_expected, int byte1, int byte2, int byte3, int byte4);
tests/src/unit-unicode3.cpp
+25 -1
View File
@@ -14,6 +14,7 @@
#include <nlohmann/json.hpp>
using nlohmann::json;
#include <algorithm>
#include <fstream>
#include <sstream>
#include <iostream>
@@ -75,8 +76,11 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
static std::string s_replaced2;
static std::string s_replaced_ascii;
static std::string s_replaced2_ascii;
static std::string s_kept;
static std::string s_kept2;
static std::string s_kept_ascii;
// dumping with ignore/replace must not throw in any case
// dumping with ignore/replace/keep must not throw in any case
s_ignored = j.dump(-1, ' ', false, json::error_handler_t::ignore);
s_ignored2 = j2.dump(-1, ' ', false, json::error_handler_t::ignore);
s_ignored_ascii = j.dump(-1, ' ', true, json::error_handler_t::ignore);
@@ -85,6 +89,9 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
s_replaced2 = j2.dump(-1, ' ', false, json::error_handler_t::replace);
s_replaced_ascii = j.dump(-1, ' ', true, json::error_handler_t::replace);
s_replaced2_ascii = j2.dump(-1, ' ', true, json::error_handler_t::replace);
s_kept = j.dump(-1, ' ', false, json::error_handler_t::keep);
s_kept2 = j2.dump(-1, ' ', false, json::error_handler_t::keep);
s_kept_ascii = j.dump(-1, ' ', true, json::error_handler_t::keep);
if (success_expected)
{
@@ -94,6 +101,7 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
// all dumps should agree on the string
CHECK(s_strict == s_ignored);
CHECK(s_strict == s_replaced);
CHECK(s_strict == s_kept);
}
else
{
@@ -105,6 +113,20 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
// check that replace string contains a replacement character
CHECK(s_replaced.find("\xEF\xBF\xBD") != std::string::npos);
// ignore drops the invalid bytes, keep copies them
CHECK(s_ignored != s_kept);
CHECK(s_ignored_ascii != s_kept_ascii);
// unless a byte needs escaping, keep copies the input unchanged
const bool needs_escaping = std::any_of(json_string.begin(), json_string.end(), [](char c)
{
return static_cast<unsigned char>(c) < 0x20 || c == '"' || c == '\\';
});
if (!needs_escaping)
{
CHECK(s_kept == "\"" + json_string + "\"");
}
}
// check that prefix and suffix are preserved
@@ -116,6 +138,8 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
CHECK(s_replaced2.substr(s_replaced2.size() - 4, 3) == "xyz");
CHECK(s_replaced2_ascii.substr(1, 3) == "abc");
CHECK(s_replaced2_ascii.substr(s_replaced2_ascii.size() - 4, 3) == "xyz");
CHECK(s_kept2.substr(1, 3) == "abc");
CHECK(s_kept2.substr(s_kept2.size() - 4, 3) == "xyz");
}
void check_utf8string(bool success_expected, int byte1, int byte2, int byte3, int byte4);
tests/src/unit-unicode4.cpp
+25 -1
View File
@@ -14,6 +14,7 @@
#include <nlohmann/json.hpp>
using nlohmann::json;
#include <algorithm>
#include <fstream>
#include <sstream>
#include <iostream>
@@ -75,8 +76,11 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
static std::string s_replaced2;
static std::string s_replaced_ascii;
static std::string s_replaced2_ascii;
static std::string s_kept;
static std::string s_kept2;
static std::string s_kept_ascii;
// dumping with ignore/replace must not throw in any case
// dumping with ignore/replace/keep must not throw in any case
s_ignored = j.dump(-1, ' ', false, json::error_handler_t::ignore);
s_ignored2 = j2.dump(-1, ' ', false, json::error_handler_t::ignore);
s_ignored_ascii = j.dump(-1, ' ', true, json::error_handler_t::ignore);
@@ -85,6 +89,9 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
s_replaced2 = j2.dump(-1, ' ', false, json::error_handler_t::replace);
s_replaced_ascii = j.dump(-1, ' ', true, json::error_handler_t::replace);
s_replaced2_ascii = j2.dump(-1, ' ', true, json::error_handler_t::replace);
s_kept = j.dump(-1, ' ', false, json::error_handler_t::keep);
s_kept2 = j2.dump(-1, ' ', false, json::error_handler_t::keep);
s_kept_ascii = j.dump(-1, ' ', true, json::error_handler_t::keep);
if (success_expected)
{
@@ -94,6 +101,7 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
// all dumps should agree on the string
CHECK(s_strict == s_ignored);
CHECK(s_strict == s_replaced);
CHECK(s_strict == s_kept);
}
else
{
@@ -105,6 +113,20 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
// check that replace string contains a replacement character
CHECK(s_replaced.find("\xEF\xBF\xBD") != std::string::npos);
// ignore drops the invalid bytes, keep copies them
CHECK(s_ignored != s_kept);
CHECK(s_ignored_ascii != s_kept_ascii);
// unless a byte needs escaping, keep copies the input unchanged
const bool needs_escaping = std::any_of(json_string.begin(), json_string.end(), [](char c)
{
return static_cast<unsigned char>(c) < 0x20 || c == '"' || c == '\\';
});
if (!needs_escaping)
{
CHECK(s_kept == "\"" + json_string + "\"");
}
}
// check that prefix and suffix are preserved
@@ -116,6 +138,8 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
CHECK(s_replaced2.substr(s_replaced2.size() - 4, 3) == "xyz");
CHECK(s_replaced2_ascii.substr(1, 3) == "abc");
CHECK(s_replaced2_ascii.substr(s_replaced2_ascii.size() - 4, 3) == "xyz");
CHECK(s_kept2.substr(1, 3) == "abc");
CHECK(s_kept2.substr(s_kept2.size() - 4, 3) == "xyz");
}
void check_utf8string(bool success_expected, int byte1, int byte2, int byte3, int byte4);
tests/src/unit-unicode5.cpp
+25 -1
View File
@@ -14,6 +14,7 @@
#include <nlohmann/json.hpp>
using nlohmann::json;
#include <algorithm>
#include <fstream>
#include <sstream>
#include <iostream>
@@ -75,8 +76,11 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
static std::string s_replaced2;
static std::string s_replaced_ascii;
static std::string s_replaced2_ascii;
static std::string s_kept;
static std::string s_kept2;
static std::string s_kept_ascii;
// dumping with ignore/replace must not throw in any case
// dumping with ignore/replace/keep must not throw in any case
s_ignored = j.dump(-1, ' ', false, json::error_handler_t::ignore);
s_ignored2 = j2.dump(-1, ' ', false, json::error_handler_t::ignore);
s_ignored_ascii = j.dump(-1, ' ', true, json::error_handler_t::ignore);
@@ -85,6 +89,9 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
s_replaced2 = j2.dump(-1, ' ', false, json::error_handler_t::replace);
s_replaced_ascii = j.dump(-1, ' ', true, json::error_handler_t::replace);
s_replaced2_ascii = j2.dump(-1, ' ', true, json::error_handler_t::replace);
s_kept = j.dump(-1, ' ', false, json::error_handler_t::keep);
s_kept2 = j2.dump(-1, ' ', false, json::error_handler_t::keep);
s_kept_ascii = j.dump(-1, ' ', true, json::error_handler_t::keep);
if (success_expected)
{
@@ -94,6 +101,7 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
// all dumps should agree on the string
CHECK(s_strict == s_ignored);
CHECK(s_strict == s_replaced);
CHECK(s_strict == s_kept);
}
else
{
@@ -105,6 +113,20 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
// check that replace string contains a replacement character
CHECK(s_replaced.find("\xEF\xBF\xBD") != std::string::npos);
// ignore drops the invalid bytes, keep copies them
CHECK(s_ignored != s_kept);
CHECK(s_ignored_ascii != s_kept_ascii);
// unless a byte needs escaping, keep copies the input unchanged
const bool needs_escaping = std::any_of(json_string.begin(), json_string.end(), [](char c)
{
return static_cast<unsigned char>(c) < 0x20 || c == '"' || c == '\\';
});
if (!needs_escaping)
{
CHECK(s_kept == "\"" + json_string + "\"");
}
}
// check that prefix and suffix are preserved
@@ -116,6 +138,8 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
CHECK(s_replaced2.substr(s_replaced2.size() - 4, 3) == "xyz");
CHECK(s_replaced2_ascii.substr(1, 3) == "abc");
CHECK(s_replaced2_ascii.substr(s_replaced2_ascii.size() - 4, 3) == "xyz");
CHECK(s_kept2.substr(1, 3) == "abc");
CHECK(s_kept2.substr(s_kept2.size() - 4, 3) == "xyz");
}
void check_utf8string(bool success_expected, int byte1, int byte2, int byte3, int byte4);