From 2aecd86b37340e3aec7b169e1c205ff0417b7351 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Thu, 29 Oct 2026 07:37:33 +0100 Subject: [PATCH] Document the contract of object keys passed to json_sax::key and key callbacks (#5956) * Document the contract of object keys passed to json_sax::key and key callbacks Describe what the name passed to json_sax::key and to a parser callback for parse_event_t::key holds when parsing JSON text: escapes are decoded to UTF-8, different spellings of a name yield identical strings, the name is validated before the call (invalid escapes or UTF-8 give a parse error and no key event), no normalization or case conversion is applied, the event precedes the ':' check, and duplicate names are reported once per occurrence. Also note that sax_parse passes binary-format keys unchanged. See discussion #5955. Signed-off-by: Niels Lohmann * Add a test for the object key contract and correct the BON8 note Pin the guarantees documented for json_sax::key and the key event of parser callbacks: decoded names, validation before delivery, delivery before the colon, one event per repeated name, and the binary formats. BON8 rejects ill-formed UTF-8 in a name even with sax_parse, so the note no longer lists it with the formats that pass bytes through. See discussion #5955. Signed-off-by: Niels Lohmann * Fix the GCC build of the object key contract test GCC 11 in C++11 mode rejects a default argument on a lambda parameter (-Wc++14-extensions) and ignores a static_cast on the warn_unused_result of parse(). Pass the input format explicitly and bind the parse result to a variable. Signed-off-by: Niels Lohmann * Add version history for the object key contract Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann --- .../docs/api/basic_json/parser_callback_t.md | 9 ++ docs/mkdocs/docs/api/json_sax/key.md | 21 +++ .../docs/features/parsing/sax_interface.md | 3 + tests/src/unit-class_parser.cpp | 136 ++++++++++++++++++ 4 files changed, 169 insertions(+) diff --git a/docs/mkdocs/docs/api/basic_json/parser_callback_t.md b/docs/mkdocs/docs/api/basic_json/parser_callback_t.md index a12634b9d..643610cb2 100644 --- a/docs/mkdocs/docs/api/basic_json/parser_callback_t.md +++ b/docs/mkdocs/docs/api/basic_json/parser_callback_t.md @@ -63,6 +63,15 @@ called: Whether the JSON value which called the function during parsing should be kept (`#!cpp true`) or not (`#!cpp false`). In the latter case, it is skipped completely, or replaced by `null` if it is the top-level value. +## Notes + +For a `parse_event_t::key` event, `parsed` holds the decoded object name, with the same guarantees as the `val` +parameter of [`json_sax::key`](../json_sax/key.md#notes): all escape sequences are decoded and `\uXXXX` escapes are +stored as UTF-8, so different spellings of the same name yield identical strings; the name is validated completely +before the callback is called, so a name with an invalid escape sequence or ill-formed UTF-8 causes a parse error and +no `key` event; no Unicode normalization or case conversion is applied; and a name that occurs several times in the +same object yields one `key` event per occurrence. + ## Examples ??? example "Example: skip an object key while parsing" diff --git a/docs/mkdocs/docs/api/json_sax/key.md b/docs/mkdocs/docs/api/json_sax/key.md index 591f590e0..14b1b52a1 100644 --- a/docs/mkdocs/docs/api/json_sax/key.md +++ b/docs/mkdocs/docs/api/json_sax/key.md @@ -19,6 +19,23 @@ Whether parsing should proceed. It is safe to move the passed object key value. +When parsing JSON text, `val` is the decoded object name, not its spelling in the input: + +- All escape sequences (`\"`, `\\`, `\/`, `\b`, `\f`, `\n`, `\r`, `\t`, `\uXXXX`, and surrogate pairs) are + decoded, and `\uXXXX` escapes are stored as UTF-8. Different spellings of the same name, such as `#!json "/"` + and `#!json "\/"`, or `#!json "\u00e9"` and `#!json "\u00E9"`, yield identical strings. An escaped `\u0000` yields + an embedded NUL byte. +- The name is scanned and validated completely before `key` is called. If it contains an invalid escape sequence, an + unpaired surrogate, a control character, or ill-formed UTF-8, the parse fails with a + [`parse_error`](../../home/exceptions.md#parse-errors), and `key` is not called for it. +- No Unicode normalization or case conversion is applied: names that differ in any byte are different strings. +- `key` is called before the following `:` is read, so a parse error may still follow for the same member. +- A name that occurs several times in the same object is passed once per occurrence. + +For the binary formats, object keys contain no escape sequences. For CBOR, MessagePack, UBJSON, and BSON, `val` holds +the bytes stored in the input, and [`sax_parse`](../basic_json/sax_parse.md) passes them unchanged, without checking them +for valid UTF-8. BON8 encodes characters itself, so an ill-formed name is a parse error and `key` is not called for it. + ## Examples ??? example @@ -45,3 +62,7 @@ It is safe to move the passed object key value. ## Version history - Added in version 3.2.0. +- The behavior described in the notes above (decoded names, validation before `key` is called, no normalization, `key` + before the `:` check, one call per duplicate, bytes of binary formats passed unchanged) is the same in all versions + from 3.2.0 on, and for the `key` event of a [parser callback](../basic_json/parser_callback_t.md) from 3.0.0 on. BSON + is only supported from version 3.4.0, and BON8 only from version 3.13.0. diff --git a/docs/mkdocs/docs/features/parsing/sax_interface.md b/docs/mkdocs/docs/features/parsing/sax_interface.md index 872105a20..f732f4b2e 100644 --- a/docs/mkdocs/docs/features/parsing/sax_interface.md +++ b/docs/mkdocs/docs/features/parsing/sax_interface.md @@ -64,6 +64,9 @@ bool parse_error(std::size_t position, const std::string& last_token, const json The return value of each function determines whether parsing should proceed. +Object keys are passed to `key` decoded and validated: escape sequences are already resolved, so different spellings of +the same name yield the same string. See [`key`](../../api/json_sax/key.md#notes) for the exact guarantees. + ## Stepping through the events The events for a small input, step by step, and what two different handlers do with them. Press *Play* or step with *Next*: diff --git a/tests/src/unit-class_parser.cpp b/tests/src/unit-class_parser.cpp index dd78f7ff4..1662fd037 100644 --- a/tests/src/unit-class_parser.cpp +++ b/tests/src/unit-class_parser.cpp @@ -3030,3 +3030,139 @@ TEST_CASE("a json_sax_dom_parser keeps the values read before a parse error") CHECK(read_before_error(c.first) == c.second); } } + +TEST_CASE("object keys are delivered decoded and validated") +{ + // json_sax::key and the key event of a parser callback see the decoded name, and + // only after the whole name was scanned and validated + const auto keys_of_format = [](const std::string & text, const json::input_format_t format) + { + SaxEventLogger l; + json::sax_parse(text, &l, format); + std::vector keys; + for (const auto& e : l.events) + { + if (e.compare(0, 4, "key(") == 0) + { + keys.push_back(e.substr(4, e.size() - 5)); + } + } + return keys; + }; + + const auto keys_of = [&keys_of_format](const std::string & text) + { + return keys_of_format(text, json::input_format_t::json); + }; + + const auto callback_keys_of = [](const std::string & text) + { + std::vector keys; + const json::parser_callback_t cb = [&keys](int /*depth*/, json::parse_event_t event, json & parsed) + { + if (event == json::parse_event_t::key) + { + keys.push_back(parsed.get()); + } + return true; + }; + const json parsed = json::parse(text, cb, false); + static_cast(parsed); + return keys; + }; + + SECTION("different spellings of a name yield the same string") + { + const std::string e_acute = "\xc3\xa9"; + const std::string text = std::string(R"({"/":1,"\/":2,"\u002f":3,"\u00e9":4,"\u00E9":5,")") + e_acute + R"(":6})"; + const std::vector expected = {"/", "/", "/", e_acute, e_acute, e_acute}; + CHECK(keys_of(text) == expected); + CHECK(callback_keys_of(text) == expected); + } + + SECTION("escape sequences are decoded") + { + const std::string simple = R"({"\"\\\/\b\f\n\r\t":1})"; + const std::vector simple_expected = {std::string("\"\\/\b\f\n\r\t")}; + CHECK(keys_of(simple) == simple_expected); + CHECK(callback_keys_of(simple) == simple_expected); + + const std::string nul = R"({"\u0000":1})"; + const std::vector nul_expected = {std::string(1, '\0')}; + CHECK(keys_of(nul) == nul_expected); + CHECK(callback_keys_of(nul) == nul_expected); + + const std::string pair = R"({"\ud83d\ude00":1})"; + const std::vector pair_expected = {"\xf0\x9f\x98\x80"}; + CHECK(keys_of(pair) == pair_expected); + CHECK(callback_keys_of(pair) == pair_expected); + } + + SECTION("no normalization and no case conversion") + { + const std::string text = std::string("{\"\xc3\xa9\":1,\"e\xcc\x81\":2,\"A\":3,\"a\":4}"); + const std::vector expected = {"\xc3\xa9", "e\xcc\x81", "A", "a"}; + CHECK(keys_of(text) == expected); + CHECK(callback_keys_of(text) == expected); + } + + SECTION("a name that is not valid is not delivered") + { + const std::vector texts = + { + R"({"\u12G4":1})", // malformed escape + R"({"\udc00":1})", // lone low surrogate + R"({"\ud800":1})", // lone high surrogate + R"({"\ud800\u0041":1})", // high surrogate not followed by a low one + R"({"\x":1})", // unknown escape + std::string("{\"a\x01\":1}"), // control character + std::string("{\"\xff\":1}") // ill-formed UTF-8 + }; + for (const auto& text : texts) + { + CAPTURE(text) + CHECK(!json::accept(text)); + CHECK(keys_of(text).empty()); + CHECK(callback_keys_of(text).empty()); + } + } + + SECTION("a name is delivered before the following colon is read") + { + const std::string text = R"({"a" 1})"; + const std::vector expected = {"a"}; + CHECK(!json::accept(text)); + CHECK(keys_of(text) == expected); + CHECK(callback_keys_of(text) == expected); + } + + SECTION("a repeated name is delivered once per occurrence") + { + const std::string text = R"({"a":1,"a":2})"; + const std::vector expected = {"a", "a"}; + CHECK(keys_of(text) == expected); + CHECK(callback_keys_of(text) == expected); + } + + SECTION("binary formats pass the stored bytes through") + { + const std::string stored = "\xff\\"; + const std::vector expected = {stored}; + const auto keys_of_bytes = [&keys_of_format](const std::vector& v, json::input_format_t format) + { + return keys_of_format(std::string(v.begin(), v.end()), format); + }; + + CHECK(keys_of_bytes({0xA1, 0x62, 0xFF, 0x5C, 0x01}, json::input_format_t::cbor) == expected); + CHECK(keys_of_bytes({0x81, 0xA2, 0xFF, 0x5C, 0x01}, json::input_format_t::msgpack) == expected); + CHECK(keys_of_bytes({'{', 'i', 2, 0xFF, 0x5C, 'i', 1, '}'}, json::input_format_t::ubjson) == expected); + CHECK(keys_of_bytes({0x0D, 0, 0, 0, 0x10, 0xFF, 0x5C, 0, 1, 0, 0, 0, 0}, json::input_format_t::bson) == expected); + } + + SECTION("BON8 rejects ill-formed UTF-8 in a name") + { + const std::vector valid = {"\\"}; + CHECK(keys_of_format(std::string("\x87\x5C\x91"), json::input_format_t::bon8) == valid); + CHECK(keys_of_format(std::string("\x87\xC3\x28\x91"), json::input_format_t::bon8).empty()); + } +}