From 9ba25b1b66aed72148c6767ae7c809b9cb6c3d17 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Fri, 30 Oct 2026 22:34:32 +0100 Subject: [PATCH] json_view: accept() checks strings without decoding them The builder that accept() runs stores no nodes, but still decoded every string with escapes into the arena (slow_string(): memcpy of the runs between escapes and the UTF-8 of every \u escape). It now calls check_string() instead, which has the same checks and the same errors and error positions (control characters, ill-formed UTF-8, bad escapes, bad hex digits, lone and unpaired surrogates, a missing closing quote) and skips the text between them with scan_string_run(), without writing anything. The builder that stores keeps slow_string(). The flag that passes the request from accept() to build() now sits next to `discarded` in document_data, where it uses padding: no member of document_data moves (the first version put it after `nodes`, which moved all members behind it by eight bytes). Tests: the new section of the accept test checks runs of text of every length from 0 to 40 between valid and invalid escapes, UTF-8 sequences and control characters, in values and in keys, cut near the end of the text, against json::accept(). Checked against a checker that accepts a lone low surrogate. Signed-off-by: Niels Lohmann --- include/nlohmann/detail/view/builder.hpp | 85 +++++++++++++++++- .../nlohmann/detail/view/document_data.hpp | 2 +- single_include/nlohmann/json_view.hpp | 87 ++++++++++++++++++- tests/src/unit-json_view_builder.cpp | 41 +++++++++ 4 files changed, 211 insertions(+), 4 deletions(-) diff --git a/include/nlohmann/detail/view/builder.hpp b/include/nlohmann/detail/view/builder.hpp index a17b21fe3..9f96cbe00 100644 --- a/include/nlohmann/detail/view/builder.hpp +++ b/include/nlohmann/detail/view/builder.hpp @@ -16,6 +16,7 @@ #include // memcmp, memcpy #include // numeric_limits #include // string +#include // false_type, integral_constant, true_type #include // vector #include @@ -246,6 +247,88 @@ class builder return p; } + /// the string from p, the first backslash, control character, ill-formed UTF-8 byte, or the end, checked and not + /// decoded (accept() stores nothing, so the arena is not written): the checks, errors and error positions of + /// slow_string(), for everything up to the closing quote + NLOHMANN_VIEW_NOINLINE decoded check_string(const unsigned char* p) + { + const unsigned char* const end = m_end; + for (;;) + { + if (p == end) + { + return failed(error_code::string_missing_quote, p); + } + const unsigned char c = *p; + if (c == '"') + { + ++p; + return decoded{p, 0, 0}; + } + if (c != '\\') + { + return failed(c < 0x20 ? error_code::string_control_character : error_code::string_utf8, p); + } + ++p; + if (p == end) + { + return failed(error_code::string_missing_quote, p); + } + const unsigned char d = *p++; + if (d == 'u') + { + const int cp = hex4(p); // (moves p past the four digits) + if (NLOHMANN_VIEW_UNLIKELY(cp < 0)) + { + return failed(error_code::string_unicode_hex, p); + } + if (NLOHMANN_VIEW_UNLIKELY((cp & 0xF800) == 0xD800)) // a surrogate + { + if (cp >= 0xDC00) + { + return failed(error_code::string_surrogate_low, p); + } + if (end - p < 2 || p[0] != '\\' || p[1] != 'u') + { + return failed(error_code::string_surrogate_high, p); + } + p += 2; + const int lo = hex4(p); // (moves p past the four digits) + if (lo < 0) + { + return failed(error_code::string_unicode_hex, p); + } + if (lo < 0xDC00 || lo > 0xDFFF) + { + return failed(error_code::string_surrogate_high, p); + } + } + } + else if (!(d == '"' || d == '\\' || d == '/' || d == 'b' || d == 'f' || d == 'n' || d == 'r' || d == 't')) + { + --p; + return failed(error_code::string_escape, p); + } + p = scan_string_run(p, end); // the text up to the next backslash, quote, control character, or ill-formed UTF-8 + } + } + + /// the string from p, decoded into the arena, or only checked (Store == false) + decoded decode_string(const unsigned char* s, const unsigned char* p) + { + return decode_string(s, p, std::integral_constant()); + } + + decoded decode_string(const unsigned char* s, const unsigned char* p, std::true_type /*unused*/) + { + return slow_string(s, p); + } + + decoded decode_string(const unsigned char* /*unused*/, const unsigned char* p, std::false_type /*unused*/) + { + return check_string(p); + } + /// s points to the start of the string, p to a backslash, a control /// character, an ill-formed UTF-8 byte, or the end: decode the string into /// the arena @@ -1193,7 +1276,7 @@ indent_done: ++p; return true; } - const decoded r = cold.slow_string(s, p); + const decoded r = cold.decode_string(s, p); if (r.p == nullptr) { return false; diff --git a/include/nlohmann/detail/view/document_data.hpp b/include/nlohmann/detail/view/document_data.hpp index fd0d6db44..7cf902237 100644 --- a/include/nlohmann/detail/view/document_data.hpp +++ b/include/nlohmann/detail/view/document_data.hpp @@ -36,7 +36,6 @@ struct document_data const char* src = nullptr; std::size_t size = 0; node* nodes = nullptr; ///< the node index (see node.hpp) - bool validate_only = false; ///< accept(): build() checks the text and stores no nodes std::size_t node_count = 0; std::size_t node_capacity = 0; node* inline_nodes = nullptr; ///< node array allocated together with this header @@ -59,6 +58,7 @@ struct document_data std::vector large_objects; ///< positions of the objects to index (noted while parsing) std::array base = {{nullptr, nullptr, nullptr, nullptr}}; ///< string bases: source, arena, edit arena, unused (indexed by flags & node_flags::storage) bool discarded = true; + bool validate_only = false; ///< accept(): build() checks the text and stores no nodes (next to `discarded`: no member moves) /// The storage of edits (editable documents only; see edit_storage.hpp). /// No edit moves or resizes the parsed index, and the nodes of values diff --git a/single_include/nlohmann/json_view.hpp b/single_include/nlohmann/json_view.hpp index 527d3aeea..24938d0b3 100644 --- a/single_include/nlohmann/json_view.hpp +++ b/single_include/nlohmann/json_view.hpp @@ -88,6 +88,7 @@ #include // memcmp, memcpy #include // numeric_limits #include // string +#include // false_type, integral_constant, true_type #include // vector // #include @@ -366,7 +367,6 @@ struct document_data const char* src = nullptr; std::size_t size = 0; node* nodes = nullptr; ///< the node index (see node.hpp) - bool validate_only = false; ///< accept(): build() checks the text and stores no nodes std::size_t node_count = 0; std::size_t node_capacity = 0; node* inline_nodes = nullptr; ///< node array allocated together with this header @@ -389,6 +389,7 @@ struct document_data std::vector large_objects; ///< positions of the objects to index (noted while parsing) std::array base = {{nullptr, nullptr, nullptr, nullptr}}; ///< string bases: source, arena, edit arena, unused (indexed by flags & node_flags::storage) bool discarded = true; + bool validate_only = false; ///< accept(): build() checks the text and stores no nodes (next to `discarded`: no member moves) /// The storage of edits (editable documents only; see edit_storage.hpp). /// No edit moves or resizes the parsed index, and the nodes of values @@ -1489,6 +1490,88 @@ class builder return p; } + /// the string from p, the first backslash, control character, ill-formed UTF-8 byte, or the end, checked and not + /// decoded (accept() stores nothing, so the arena is not written): the checks, errors and error positions of + /// slow_string(), for everything up to the closing quote + NLOHMANN_VIEW_NOINLINE decoded check_string(const unsigned char* p) + { + const unsigned char* const end = m_end; + for (;;) + { + if (p == end) + { + return failed(error_code::string_missing_quote, p); + } + const unsigned char c = *p; + if (c == '"') + { + ++p; + return decoded{p, 0, 0}; + } + if (c != '\\') + { + return failed(c < 0x20 ? error_code::string_control_character : error_code::string_utf8, p); + } + ++p; + if (p == end) + { + return failed(error_code::string_missing_quote, p); + } + const unsigned char d = *p++; + if (d == 'u') + { + const int cp = hex4(p); // (moves p past the four digits) + if (NLOHMANN_VIEW_UNLIKELY(cp < 0)) + { + return failed(error_code::string_unicode_hex, p); + } + if (NLOHMANN_VIEW_UNLIKELY((cp & 0xF800) == 0xD800)) // a surrogate + { + if (cp >= 0xDC00) + { + return failed(error_code::string_surrogate_low, p); + } + if (end - p < 2 || p[0] != '\\' || p[1] != 'u') + { + return failed(error_code::string_surrogate_high, p); + } + p += 2; + const int lo = hex4(p); // (moves p past the four digits) + if (lo < 0) + { + return failed(error_code::string_unicode_hex, p); + } + if (lo < 0xDC00 || lo > 0xDFFF) + { + return failed(error_code::string_surrogate_high, p); + } + } + } + else if (!(d == '"' || d == '\\' || d == '/' || d == 'b' || d == 'f' || d == 'n' || d == 'r' || d == 't')) + { + --p; + return failed(error_code::string_escape, p); + } + p = scan_string_run(p, end); // the text up to the next backslash, quote, control character, or ill-formed UTF-8 + } + } + + /// the string from p, decoded into the arena, or only checked (Store == false) + decoded decode_string(const unsigned char* s, const unsigned char* p) + { + return decode_string(s, p, std::integral_constant()); + } + + decoded decode_string(const unsigned char* s, const unsigned char* p, std::true_type /*unused*/) + { + return slow_string(s, p); + } + + decoded decode_string(const unsigned char* /*unused*/, const unsigned char* p, std::false_type /*unused*/) + { + return check_string(p); + } + /// s points to the start of the string, p to a backslash, a control /// character, an ill-formed UTF-8 byte, or the end: decode the string into /// the arena @@ -2436,7 +2519,7 @@ indent_done: ++p; return true; } - const decoded r = cold.slow_string(s, p); + const decoded r = cold.decode_string(s, p); if (r.p == nullptr) { return false; diff --git a/tests/src/unit-json_view_builder.cpp b/tests/src/unit-json_view_builder.cpp index 0071280ae..c0968d59e 100644 --- a/tests/src/unit-json_view_builder.cpp +++ b/tests/src/unit-json_view_builder.cpp @@ -509,6 +509,47 @@ TEST_CASE("json_view builder") CAPTURE(text) CHECK(nlohmann::basic_json_document::accept(text) == float_json::accept(text)); } + + // strings are checked and not decoded: runs of text of every length from 0 to 40 between valid and invalid + // escapes, UTF-8 sequences and control characters, in values and in keys, cut near the end of the text + const std::array valid = {{"\\n", "\\\"", "\\\\", "\\/", "\\t", "\\u00e4", "\\u20ac", "\\ud83d\\ude00", "\\u0000"}}; + const std::array invalid = {{"\\x", "\\ud800", "\\udc00", "\\u12G4", "\\u12", "\\ud83dx", "\\ud83d\\u0041", "\\ud83d\\ud83d", "\\", "\x01", "\x1f", "\x7f"}}; + const std::array utf8 = {{"\xc3\xa4", "\xe2\x82\xac", "\xf0\x9f\x98\x80", "\xc3", "\xff", "\x80"}}; + std::mt19937 rng(2029); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed): reproducible + for (std::size_t run = 0; run <= 40; ++run) + { + for (int i = 0; i < 30; ++i) + { + std::string content = std::string(rng() % 20, 'a'); + const auto pieces = 1 + (rng() % 6); + for (unsigned k = 0; k < pieces; ++k) + { + content += valid[rng() % valid.size()]; + content += std::string(k == 0 ? run : rng() % 41, static_cast('b' + (rng() % 20))); + switch (rng() % 8) + { + case 0: + content += utf8[rng() % 3]; + break; + case 1: + content += utf8[rng() % utf8.size()]; + break; + case 2: + content += invalid[rng() % invalid.size()]; + break; + default: + break; + } + } + for (const std::string& text : + { + "[\"" + content + "\"]", "{\"" + content + "\":1}", "[\"" + content, "[\"" + content.substr(0, content.size() - (rng() % 4)) + "\"]" + }) + { + check_accept(text); + } + } + } } SECTION("test files")