From 6d4543e743a30d7330b6a6afcd13751207df2998 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Tue, 6 Oct 2026 22:29:13 +0200 Subject: [PATCH 01/12] Fix destroy() for ObjectTypes without reverse iteration (#5767) The non-recursive destroy walk from #5762 picked an object's last child via object_t::rbegin() and std::prev(end()). Neither is available for every ObjectType: no_key_compare_map in unit-custom-object-type.cpp has no rbegin(), so develop no longer compiles that test, and hash maps such as std::unordered_map only have forward iterators. The walk can take an object's children in any order, as long as it finds the same child again while the object is not modified in between. So objects with bidirectional iterators keep using their last child (O(1) to remove from vector-based maps like ordered_map), and objects with forward-only iterators use begin() instead. No reverse iteration or rbegin() is needed any more, and the walk stays allocation-free. Adds a forward-only ObjectType to the tests, destroyed both with mixed nesting and 100000 levels deep. Signed-off-by: Niels Lohmann --- include/nlohmann/json.hpp | 77 ++++++--- single_include/nlohmann/json.hpp | 77 ++++++--- tests/src/unit-custom-object-type.cpp | 237 ++++++++++++++++++++++++++ 3 files changed, 343 insertions(+), 48 deletions(-) diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index aa8b1a6c6..f698163fe 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -638,21 +638,49 @@ private: } } - static basic_json& last_child(basic_json& v) + // The walk in destroy_container() may take the children of a + // container in any order, as long as it picks the same child again + // while that container is not modified in between. Arrays and + // objects with bidirectional iterators (std::map, ordered_map, ...) + // use their last child, which a vector-based container can remove + // in O(1). ObjectType only needs forward iterators, though (e.g. + // std::unordered_map), so other objects use their first child. + template + static typename ObjectType_::iterator walk_child_it(ObjectType_& o, std::bidirectional_iterator_tag /*unused*/) + { + return std::prev(o.end()); + } + + template + static typename ObjectType_::iterator walk_child_it(ObjectType_& o, std::forward_iterator_tag /*unused*/) + { + return o.begin(); + } + + template + static typename ObjectType_::iterator walk_child_it(ObjectType_& o) + { + JSON_ASSERT(!o.empty()); + return walk_child_it(o, typename std::iterator_traits::iterator_category()); + } + + // the child of a non-empty array/object v that the walk in + // destroy_container() continues with (see walk_child_it() above) + static basic_json& walk_child(basic_json& v) { if (v.m_data.m_type == value_t::array) { return v.m_data.m_value.array->back(); } JSON_ASSERT(v.m_data.m_type == value_t::object); - return v.m_data.m_value.object->rbegin()->second; + return walk_child_it(*v.m_data.m_value.object)->second; } - // removes the last child of a non-empty array/object v; this never + // removes walk_child(v) from a non-empty array/object v; this never // allocates, and since it is only ever called when that child is a // scalar or an already-empty array/object, destroying it never // recurses more than one level deep (see destroy() below) - static void pop_last_child(basic_json& v) + static void pop_walk_child(basic_json& v) { if (v.m_data.m_type == value_t::array) { @@ -661,9 +689,7 @@ private: else { JSON_ASSERT(v.m_data.m_type == value_t::object); - // erase() needs a forward iterator, so std::prev(end()) is - // used here rather than rbegin() (see last_child() above) - v.m_data.m_value.object->erase(std::prev(v.m_data.m_value.object->end())); + v.m_data.m_value.object->erase(walk_child_it(*v.m_data.m_value.object)); } } @@ -734,14 +760,17 @@ public: // bad_alloc, which would escape this noexcept destructor and // terminate the program (#5135). // - // Instead, walk down the "last child" chain, reversing links - // as we go: cur is the container currently being emptied, - // and prev is its parent (value_t::null when there is none). - // Each parent's last child slot doubles as storage for that - // parent's own parent link while we are below it, so no - // extra memory is needed. We only ever remove a child once - // it is a scalar or an empty array/object, which neither - // allocates nor recurses more than one level deep. + // Instead, walk down a chain of children (always the one + // walk_child() picks), reversing links as we go: cur is the + // container currently being emptied, and prev is its parent + // (value_t::null when there is none). Each parent's + // walk_child() slot doubles as storage for that parent's own + // parent link while we are below it, so no extra memory is + // needed; the parent is not modified meanwhile, so + // walk_child() finds that same slot again on the way up. We + // only ever remove a child once it is a scalar or an empty + // array/object, which neither allocates nor recurses more + // than one level deep. // // This json_value is not itself a basic_json, so the // top-level container is first moved into a local stand-in @@ -767,11 +796,11 @@ public: } // ascend: detach the grandparent link from prev's - // last slot, drop that (now null) slot, free cur + // walk_child() slot, drop that (now null) slot, free cur // (it is empty), then move up one level basic_json gp; - take(gp, last_child(prev)); - pop_last_child(prev); + take(gp, walk_child(prev)); + pop_walk_child(prev); free_container(cur); @@ -780,21 +809,21 @@ public: continue; } - basic_json& cur_last_ref = last_child(cur); + basic_json& cur_child_ref = walk_child(cur); - if (has_no_children(cur_last_ref)) + if (has_no_children(cur_child_ref)) { // scalar, or already-empty array/object - pop_last_child(cur); + pop_walk_child(cur); continue; } - // descend into the non-empty last child, reversing the + // descend into the non-empty child, reversing the // link: its slot takes over prev, and the child becomes // the new cur basic_json tmp; - take(tmp, cur_last_ref); - take(cur_last_ref, prev); + take(tmp, cur_child_ref); + take(cur_child_ref, prev); take(prev, cur); take(cur, tmp); } diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 00e345dd3..b6e5ca7c5 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -27877,21 +27877,49 @@ private: } } - static basic_json& last_child(basic_json& v) + // The walk in destroy_container() may take the children of a + // container in any order, as long as it picks the same child again + // while that container is not modified in between. Arrays and + // objects with bidirectional iterators (std::map, ordered_map, ...) + // use their last child, which a vector-based container can remove + // in O(1). ObjectType only needs forward iterators, though (e.g. + // std::unordered_map), so other objects use their first child. + template + static typename ObjectType_::iterator walk_child_it(ObjectType_& o, std::bidirectional_iterator_tag /*unused*/) + { + return std::prev(o.end()); + } + + template + static typename ObjectType_::iterator walk_child_it(ObjectType_& o, std::forward_iterator_tag /*unused*/) + { + return o.begin(); + } + + template + static typename ObjectType_::iterator walk_child_it(ObjectType_& o) + { + JSON_ASSERT(!o.empty()); + return walk_child_it(o, typename std::iterator_traits::iterator_category()); + } + + // the child of a non-empty array/object v that the walk in + // destroy_container() continues with (see walk_child_it() above) + static basic_json& walk_child(basic_json& v) { if (v.m_data.m_type == value_t::array) { return v.m_data.m_value.array->back(); } JSON_ASSERT(v.m_data.m_type == value_t::object); - return v.m_data.m_value.object->rbegin()->second; + return walk_child_it(*v.m_data.m_value.object)->second; } - // removes the last child of a non-empty array/object v; this never + // removes walk_child(v) from a non-empty array/object v; this never // allocates, and since it is only ever called when that child is a // scalar or an already-empty array/object, destroying it never // recurses more than one level deep (see destroy() below) - static void pop_last_child(basic_json& v) + static void pop_walk_child(basic_json& v) { if (v.m_data.m_type == value_t::array) { @@ -27900,9 +27928,7 @@ private: else { JSON_ASSERT(v.m_data.m_type == value_t::object); - // erase() needs a forward iterator, so std::prev(end()) is - // used here rather than rbegin() (see last_child() above) - v.m_data.m_value.object->erase(std::prev(v.m_data.m_value.object->end())); + v.m_data.m_value.object->erase(walk_child_it(*v.m_data.m_value.object)); } } @@ -27973,14 +27999,17 @@ public: // bad_alloc, which would escape this noexcept destructor and // terminate the program (#5135). // - // Instead, walk down the "last child" chain, reversing links - // as we go: cur is the container currently being emptied, - // and prev is its parent (value_t::null when there is none). - // Each parent's last child slot doubles as storage for that - // parent's own parent link while we are below it, so no - // extra memory is needed. We only ever remove a child once - // it is a scalar or an empty array/object, which neither - // allocates nor recurses more than one level deep. + // Instead, walk down a chain of children (always the one + // walk_child() picks), reversing links as we go: cur is the + // container currently being emptied, and prev is its parent + // (value_t::null when there is none). Each parent's + // walk_child() slot doubles as storage for that parent's own + // parent link while we are below it, so no extra memory is + // needed; the parent is not modified meanwhile, so + // walk_child() finds that same slot again on the way up. We + // only ever remove a child once it is a scalar or an empty + // array/object, which neither allocates nor recurses more + // than one level deep. // // This json_value is not itself a basic_json, so the // top-level container is first moved into a local stand-in @@ -28006,11 +28035,11 @@ public: } // ascend: detach the grandparent link from prev's - // last slot, drop that (now null) slot, free cur + // walk_child() slot, drop that (now null) slot, free cur // (it is empty), then move up one level basic_json gp; - take(gp, last_child(prev)); - pop_last_child(prev); + take(gp, walk_child(prev)); + pop_walk_child(prev); free_container(cur); @@ -28019,21 +28048,21 @@ public: continue; } - basic_json& cur_last_ref = last_child(cur); + basic_json& cur_child_ref = walk_child(cur); - if (has_no_children(cur_last_ref)) + if (has_no_children(cur_child_ref)) { // scalar, or already-empty array/object - pop_last_child(cur); + pop_walk_child(cur); continue; } - // descend into the non-empty last child, reversing the + // descend into the non-empty child, reversing the // link: its slot takes over prev, and the child becomes // the new cur basic_json tmp; - take(tmp, cur_last_ref); - take(cur_last_ref, prev); + take(tmp, cur_child_ref); + take(cur_child_ref, prev); take(prev, cur); take(cur, tmp); } diff --git a/tests/src/unit-custom-object-type.cpp b/tests/src/unit-custom-object-type.cpp index a579e0b2d..0eaaf2d3c 100644 --- a/tests/src/unit-custom-object-type.cpp +++ b/tests/src/unit-custom-object-type.cpp @@ -10,7 +10,9 @@ #include +#include #include +#include #include #include #include @@ -196,6 +198,198 @@ struct void_erase_map : std::map using void_erase_json = nlohmann::basic_json; +// wraps an iterator, but only offers the LegacyForwardIterator operations, +// like the iterators of std::unordered_map and other hash maps +template +class forward_only_iterator +{ + BaseIterator m_it{}; + + public: + using iterator_category = std::forward_iterator_tag; + using value_type = typename std::iterator_traits::value_type; + using difference_type = typename std::iterator_traits::difference_type; + using pointer = typename std::iterator_traits::pointer; + using reference = typename std::iterator_traits::reference; + + forward_only_iterator() = default; + explicit forward_only_iterator(BaseIterator it) : m_it(it) {} + + BaseIterator base() const + { + return m_it; + } + + reference operator*() const + { + return *m_it; + } + pointer operator->() const + { + return &*m_it; + } + forward_only_iterator& operator++() + { + ++m_it; + return *this; + } + forward_only_iterator operator++(int) + { + auto result = *this; + ++m_it; + return result; + } + + friend bool operator==(const forward_only_iterator& lhs, const forward_only_iterator& rhs) + { + return lhs.m_it == rhs.m_it; + } + friend bool operator!=(const forward_only_iterator& lhs, const forward_only_iterator& rhs) + { + return lhs.m_it != rhs.m_it; + } +}; + +// An ObjectType whose iterators are forward-only, as those of hash maps are; +// it has no rbegin() and its iterators no operator--. A hash map is not used +// directly for the same reason as in no_key_compare_map above. +template +class forward_only_map +{ + using map_t = std::map; + map_t data; + + public: + using key_type = typename map_t::key_type; + using mapped_type = typename map_t::mapped_type; + using value_type = typename map_t::value_type; + using size_type = typename map_t::size_type; + using allocator_type = typename map_t::allocator_type; + using iterator = forward_only_iterator; + using const_iterator = forward_only_iterator; + + forward_only_map() noexcept(std::is_nothrow_default_constructible::value) : data() {} + + template + forward_only_map(InputIt first, InputIt last) : data(first, last) {} + + iterator begin() noexcept + { + return iterator(data.begin()); + } + iterator end() noexcept + { + return iterator(data.end()); + } + const_iterator begin() const noexcept + { + return const_iterator(data.begin()); + } + const_iterator end() const noexcept + { + return const_iterator(data.end()); + } + const_iterator cbegin() const noexcept + { + return const_iterator(data.cbegin()); + } + const_iterator cend() const noexcept + { + return const_iterator(data.cend()); + } + + bool empty() const noexcept + { + return data.empty(); + } + size_type size() const noexcept + { + return data.size(); + } + size_type max_size() const noexcept + { + return data.max_size(); + } + void clear() noexcept + { + data.clear(); + } + + iterator find(const key_type& key) + { + return iterator(data.find(key)); + } + const_iterator find(const key_type& key) const + { + return const_iterator(data.find(key)); + } + size_type count(const key_type& key) const + { + return data.count(key); + } + + std::pair emplace(const key_type& key, const mapped_type& value) + { + const auto result = data.emplace(key, value); + return {iterator(result.first), result.second}; + } + + std::pair insert(const value_type& value) + { + const auto result = data.insert(value); + return {iterator(result.first), result.second}; + } + + template + void insert(InputIt first, InputIt last) + { + data.insert(first, last); + } + + mapped_type& operator[](const key_type& key) + { + return data[key]; + } + + mapped_type& at(const key_type& key) + { + return data.at(key); + } + const mapped_type& at(const key_type& key) const + { + return data.at(key); + } + + iterator erase(iterator pos) + { + return iterator(data.erase(pos.base())); + } + iterator erase(iterator first, iterator last) + { + return iterator(data.erase(first.base(), last.base())); + } + size_type erase(const key_type& key) + { + return data.erase(key); + } + + void swap(forward_only_map& other) noexcept(noexcept(data.swap(other.data))) + { + data.swap(other.data); + } + + friend bool operator==(const forward_only_map& lhs, const forward_only_map& rhs) + { + return lhs.data == rhs.data; + } + friend bool operator<(const forward_only_map& lhs, const forward_only_map& rhs) + { + return lhs.data < rhs.data; + } +}; + +using forward_only_json = nlohmann::basic_json; + } // namespace TEST_CASE("object type whose erase() returns void") @@ -322,3 +516,46 @@ TEST_CASE("object type without key_compare") } } + +TEST_CASE("object type with forward-only iterators") +{ + CHECK(std::is_same::iterator_category, + std::forward_iterator_tag>::value); + + SECTION("destroying nested objects and arrays") + { + forward_only_json j; + j["a"] = 1; + j["b"]["c"] = "x"; + j["b"]["d"] = forward_only_json::array(); + j["b"]["d"].push_back(forward_only_json::object()); + j["b"]["d"].push_back(true); + j["b"]["e"]["f"]["g"] = nullptr; + j["h"] = forward_only_json::object(); + j["i"]["j"] = 2; + + CHECK(j.size() == 4); + CHECK(j["b"].size() == 3); + CHECK(j["b"]["d"].size() == 2); + CHECK(j["b"]["e"]["f"]["g"].is_null()); + + CHECK(j.erase("b") == 1); + CHECK(j.size() == 3); + j = 42; + CHECK(j == 42); + } + + SECTION("destroying a deeply nested object") + { + constexpr std::size_t depth = 100000; + forward_only_json j; + forward_only_json* cur = &j; + for (std::size_t i = 0; i < depth; ++i) + { + (*cur)["s"] = i; + cur = &(*cur)["o"]; + } + CHECK(j["o"]["o"]["s"] == 2); + // destroyed at the end of scope without recursing per level + } +} From 9b179cee1e1813020fbd14d15bcced6b4ffee109 Mon Sep 17 00:00:00 2001 From: fhgffy <102001626+fhgffy@users.noreply.github.com> Date: Wed, 7 Oct 2026 04:29:22 +0800 Subject: [PATCH 02/12] Handle all value_t enumerators in has_no_children() (#5770) 2026-10-07: add the scalar enum labels already handled by the default branch so -Werror=switch-enum builds succeed. Keep the existing behavior and synchronize the generated single header. Signed-off-by: fhgffy <102001626+fhgffy@users.noreply.github.com> --- include/nlohmann/json.hpp | 8 ++++++++ single_include/nlohmann/json.hpp | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index f698163fe..8cf3f8397 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -633,6 +633,14 @@ private: return v.m_data.m_value.array->empty(); case value_t::object: return v.m_data.m_value.object->empty(); + case value_t::null: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: default: return true; } diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index b6e5ca7c5..9a1aaeecd 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -27872,6 +27872,14 @@ private: return v.m_data.m_value.array->empty(); case value_t::object: return v.m_data.m_value.object->empty(); + case value_t::null: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: default: return true; } From d41887511be9a00d45eb1533bc7c0854ed68c0cb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 22:29:55 +0200 Subject: [PATCH 03/12] Bump source-map-js from 1.2.1 to 1.2.2 in /docs/mkdocs/scripts/mermaid (#5769) Bumps [source-map-js](https://github.com/7rulnik/source-map-js) from 1.2.1 to 1.2.2. - [Release notes](https://github.com/7rulnik/source-map-js/releases) - [Changelog](https://github.com/7rulnik/source-map-js/blob/main/CHANGELOG.md) - [Commits](https://github.com/7rulnik/source-map-js/compare/v1.2.1...v1.2.2) --- updated-dependencies: - dependency-name: source-map-js dependency-version: 1.2.2 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- docs/mkdocs/scripts/mermaid/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/mkdocs/scripts/mermaid/package-lock.json b/docs/mkdocs/scripts/mermaid/package-lock.json index 7fbbf6226..4f9dc0395 100644 --- a/docs/mkdocs/scripts/mermaid/package-lock.json +++ b/docs/mkdocs/scripts/mermaid/package-lock.json @@ -1454,9 +1454,9 @@ } }, "node_modules/source-map-js": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", - "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.2.tgz", + "integrity": "sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw==", "license": "BSD-3-Clause", "engines": { "node": ">=0.10.0" From a7d387a762762f833a8ccfb27f4aaa34475c0849 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 22:30:22 +0200 Subject: [PATCH 04/12] Bump mkdocs-htmlproofer-plugin from 1.5.0 to 1.6.0 in /docs/mkdocs (#5771) Bumps [mkdocs-htmlproofer-plugin](https://github.com/manuzhang/mkdocs-htmlproofer-plugin) from 1.5.0 to 1.6.0. - [Release notes](https://github.com/manuzhang/mkdocs-htmlproofer-plugin/releases) - [Commits](https://github.com/manuzhang/mkdocs-htmlproofer-plugin/compare/v1.5.0...v1.6.0) --- updated-dependencies: - dependency-name: mkdocs-htmlproofer-plugin dependency-version: 1.6.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- docs/mkdocs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/mkdocs/requirements.txt b/docs/mkdocs/requirements.txt index 6396ec500..d9e7098ed 100644 --- a/docs/mkdocs/requirements.txt +++ b/docs/mkdocs/requirements.txt @@ -6,7 +6,7 @@ mkdocs-material==9.7.7 # theme for mkdocs mkdocs-material-extensions==1.3.1 # extensions mkdocs-minify-plugin==0.8.0 # plugin "minify" mkdocs-redirects==1.2.3 # plugin "redirects" -mkdocs-htmlproofer-plugin==1.5.0 # plugin "htmlproofer" +mkdocs-htmlproofer-plugin==1.6.0 # plugin "htmlproofer" mkdocs-llmstxt==0.5.0 # plugin "llmstxt" PyYAML==6.0.3 # linter From 69874e4544f2a129069ebef8d47eee40ef8c1483 Mon Sep 17 00:00:00 2001 From: fhgffy <102001626+fhgffy@users.noreply.github.com> Date: Wed, 7 Oct 2026 04:30:52 +0800 Subject: [PATCH 05/12] Fix NUL bytes in UBJSON/BJData high-precision numbers (#5760) * Fix NUL-terminated UBJSON and BJData high-precision payloads Signed-off-by: fhgffy <102001626+fhgffy@users.noreply.github.com> * Use English comments for the high-precision NUL fix Signed-off-by: fhgffy <102001626+fhgffy@users.noreply.github.com> * Track NUL bytes while reading high-precision payloads Signed-off-by: fhgffy <102001626+fhgffy@users.noreply.github.com> * Drop dates from code comments. Signed-off-by: fhgffy <102001626+fhgffy@users.noreply.github.com> * Report a NUL in a high-precision number where it is read Return the parse error from the read loop so the byte offset points at the NUL, and drop the separate check after lexing. Signed-off-by: fhgffy <102001626+fhgffy@users.noreply.github.com> * Use fixed text for the high-precision NUL error 2026-10-06: Use the known zero byte directly instead of formatting and concatenating it. Preserve the SAX token, error message, and byte offset. Signed-off-by: fhgffy <102001626+fhgffy@users.noreply.github.com> --------- Signed-off-by: fhgffy <102001626+fhgffy@users.noreply.github.com> --- .../nlohmann/detail/input/binary_reader.hpp | 6 +++++ single_include/nlohmann/json.hpp | 6 +++++ tests/src/unit-bjdata.cpp | 26 +++++++++++++++++++ tests/src/unit-ubjson.cpp | 26 +++++++++++++++++++ 4 files changed, 64 insertions(+) diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index e6c698fa0..d26e5fad4 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -3249,6 +3249,12 @@ class binary_reader { return false; } + // the lexer would stop at a NUL and accept the digits before it + if (JSON_HEDLEY_UNLIKELY(current == '\0')) + { + return sax->parse_error(chars_read, "00", parse_error::create(115, chars_read, + exception_message("invalid number text; last byte: 0x00", "high-precision number"), nullptr)); + } number_vector.push_back(static_cast(current)); } diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 9a1aaeecd..09e3b4f4a 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -16998,6 +16998,12 @@ class binary_reader { return false; } + // the lexer would stop at a NUL and accept the digits before it + if (JSON_HEDLEY_UNLIKELY(current == '\0')) + { + return sax->parse_error(chars_read, "00", parse_error::create(115, chars_read, + exception_message("invalid number text; last byte: 0x00", "high-precision number"), nullptr)); + } number_vector.push_back(static_cast(current)); } diff --git a/tests/src/unit-bjdata.cpp b/tests/src/unit-bjdata.cpp index cd989ef88..af1883827 100644 --- a/tests/src/unit-bjdata.cpp +++ b/tests/src/unit-bjdata.cpp @@ -1286,6 +1286,32 @@ TEST_CASE("BJData") CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vec2), "[json.exception.parse_error.115] parse error at byte 5: syntax error while parsing BJData high-precision number: invalid number text: 1A", json::parse_error); std::vector const vec3 = {'H', 'i', 2, '1', '.'}; CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vec3), "[json.exception.parse_error.115] parse error at byte 5: syntax error while parsing BJData high-precision number: invalid number text: 1.", json::parse_error); + // Reject NULs where they are read, including trailing NULs and payloads cut off after one. + SECTION("NUL in high-precision number (issue #5753)") + { + for (const auto& vec : std::vector> + { + {'H', 'i', 3, '1', 0, 'x'}, + {'H', 'i', 2, '1', 0}, + {'H', 'i', 3, '1', 0} + }) + { + CAPTURE(vec) + CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vec), "[json.exception.parse_error.115] parse error at byte 5: syntax error while parsing BJData high-precision number: invalid number text; last byte: 0x00", json::parse_error); + CHECK(json::from_bjdata(vec, true, false).is_discarded()); + CHECK(json::from_bjdata(vec, false, false).is_discarded()); + } + + std::vector const nested = {'[', 'H', 'i', 3, '1', 0, 'x', ']'}; + CHECK_THROWS_WITH_AS(_ = json::from_bjdata(nested), "[json.exception.parse_error.115] parse error at byte 6: syntax error while parsing BJData high-precision number: invalid number text; last byte: 0x00", json::parse_error); + CHECK(json::from_bjdata(nested, true, false).is_discarded()); + + std::vector const valid = {'H', 'i', 1, '1'}; + const auto j = json::from_bjdata(valid); + CHECK(j.is_number_unsigned()); + CHECK(j == json(1)); + } + std::vector const vec_overflow = {'H', 'i', 5, '1', 'e', '4', '0', '0'}; CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vec_overflow), "[json.exception.out_of_range.406] number overflow parsing '1e400'", json::out_of_range); std::vector const vec4 = {'H', 2, '1', '0'}; diff --git a/tests/src/unit-ubjson.cpp b/tests/src/unit-ubjson.cpp index 9a45546b6..62abd50fc 100644 --- a/tests/src/unit-ubjson.cpp +++ b/tests/src/unit-ubjson.cpp @@ -802,6 +802,32 @@ TEST_CASE("UBJSON") CHECK_THROWS_WITH_AS(_ = json::from_ubjson(vec2), "[json.exception.parse_error.115] parse error at byte 5: syntax error while parsing UBJSON high-precision number: invalid number text: 1A", json::parse_error); std::vector const vec3 = {'H', 'i', 2, '1', '.'}; CHECK_THROWS_WITH_AS(_ = json::from_ubjson(vec3), "[json.exception.parse_error.115] parse error at byte 5: syntax error while parsing UBJSON high-precision number: invalid number text: 1.", json::parse_error); + // Reject NULs where they are read, including trailing NULs and payloads cut off after one. + SECTION("NUL in high-precision number (issue #5753)") + { + for (const auto& vec : std::vector> + { + {'H', 'i', 3, '1', 0, 'x'}, + {'H', 'i', 2, '1', 0}, + {'H', 'i', 3, '1', 0} + }) + { + CAPTURE(vec) + CHECK_THROWS_WITH_AS(_ = json::from_ubjson(vec), "[json.exception.parse_error.115] parse error at byte 5: syntax error while parsing UBJSON high-precision number: invalid number text; last byte: 0x00", json::parse_error); + CHECK(json::from_ubjson(vec, true, false).is_discarded()); + CHECK(json::from_ubjson(vec, false, false).is_discarded()); + } + + std::vector const nested = {'[', 'H', 'i', 3, '1', 0, 'x', ']'}; + CHECK_THROWS_WITH_AS(_ = json::from_ubjson(nested), "[json.exception.parse_error.115] parse error at byte 6: syntax error while parsing UBJSON high-precision number: invalid number text; last byte: 0x00", json::parse_error); + CHECK(json::from_ubjson(nested, true, false).is_discarded()); + + std::vector const valid = {'H', 'i', 1, '1'}; + const auto j = json::from_ubjson(valid); + CHECK(j.is_number_unsigned()); + CHECK(j == json(1)); + } + std::vector const vec_overflow = {'H', 'i', 5, '1', 'e', '4', '0', '0'}; CHECK_THROWS_WITH_AS(_ = json::from_ubjson(vec_overflow), "[json.exception.out_of_range.406] number overflow parsing '1e400'", json::out_of_range&); std::vector const vec4 = {'H', 2, '1', '0'}; From 162e13b86f1b9641c5a12b0bad8aea066df4c6a7 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Tue, 6 Oct 2026 22:33:29 +0200 Subject: [PATCH 06/12] Add with_*_t alias templates to create basic_json types with changed template parameters (#5758) * Add helper types to make it easier to create a basic_json type with modified template parameters Signed-off-by: Niels Lohmann * Rename with_changed_*_t aliases to with_*_t and merge integer/unsigned aliases Per review discussion on #3898 between gregmarr and nlohmann: - rename with_changed_X_t to with_X_t for brevity - replace the separate with_changed_integer_t/with_changed_unsigned_t aliases with a single with_integers_t - add @sa doc comment links for the upcoming documentation page Co-authored-by: Raphael Grimm <1005058+barcode@users.noreply.github.com> Signed-off-by: Niels Lohmann * Add documentation for the with_*_t member alias templates Add docs/mkdocs/docs/api/basic_json/with_t.md documenting with_object_t, with_array_t, with_string_t, with_boolean_t, with_integers_t, with_float_t, with_allocator_t, with_json_serializer_t, with_binary_t and with_base_class_t, with an accompanying example, and link the page from the basic_json member types list and the mkdocs navigation. Co-authored-by: Raphael Grimm <1005058+barcode@users.noreply.github.com> Signed-off-by: Niels Lohmann * Add tests for the with_*_t member alias templates Check with std::is_same that each with_*_t alias produces the expected basic_json type, and that with_string_t keeps nlohmann::ordered_map as the object type when used on ordered_json. Co-authored-by: Raphael Grimm <1005058+barcode@users.noreply.github.com> Signed-off-by: Niels Lohmann * Fix with_t nav entry and document chaining of the with_*_t aliases Indent the with_t entry in mkdocs.yml so it is listed under basic_json, explain that the aliases can be chained and work on ordered_json, and test both, including json::with_object_t == ordered_json. Signed-off-by: Niels Lohmann * Add docset entry for basic_json::with_t Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann Co-authored-by: barcode Co-authored-by: Raphael Grimm <1005058+barcode@users.noreply.github.com> --- docs/docset/docSet.sql | 1 + docs/mkdocs/docs/api/basic_json/index.md | 3 + docs/mkdocs/docs/api/basic_json/with_t.md | 136 ++++++++++++++++++ .../docs/examples/json_base_class_t.cpp | 14 +- docs/mkdocs/docs/examples/with_t.cpp | 18 +++ docs/mkdocs/docs/examples/with_t.output | 1 + docs/mkdocs/mkdocs.yml | 1 + include/nlohmann/json.hpp | 64 +++++++++ single_include/nlohmann/json.hpp | 64 +++++++++ tests/src/unit-allocator.cpp | 27 +--- tests/src/unit-alt-string.cpp | 11 +- tests/src/unit-custom-base-class.cpp | 29 +--- tests/src/unit-udt.cpp | 80 ++++++++++- 13 files changed, 372 insertions(+), 77 deletions(-) create mode 100644 docs/mkdocs/docs/api/basic_json/with_t.md create mode 100644 docs/mkdocs/docs/examples/with_t.cpp create mode 100644 docs/mkdocs/docs/examples/with_t.output diff --git a/docs/docset/docSet.sql b/docs/docset/docSet.sql index 8e956413c..5549b55d0 100644 --- a/docs/docset/docSet.sql +++ b/docs/docset/docSet.sql @@ -131,6 +131,7 @@ INSERT INTO searchIndex(name, type, path) VALUES ('basic_json::to_string', 'Meth INSERT INTO searchIndex(name, type, path) VALUES ('basic_json::to_ubjson', 'Function', 'api/basic_json/to_ubjson/index.html'); INSERT INTO searchIndex(name, type, path) VALUES ('basic_json::value', 'Method', 'api/basic_json/value/index.html'); INSERT INTO searchIndex(name, type, path) VALUES ('basic_json::value_t', 'Enum', 'api/basic_json/value_t/index.html'); +INSERT INTO searchIndex(name, type, path) VALUES ('basic_json::with_t', 'Type', 'api/basic_json/with_t/index.html'); INSERT INTO searchIndex(name, type, path) VALUES ('basic_json::~basic_json', 'Method', 'api/basic_json/~basic_json/index.html'); INSERT INTO searchIndex(name, type, path) VALUES ('json', 'Class', 'api/json/index.html'); INSERT INTO searchIndex(name, type, path) VALUES ('json_pointer', 'Class', 'api/json_pointer/index.html'); diff --git a/docs/mkdocs/docs/api/basic_json/index.md b/docs/mkdocs/docs/api/basic_json/index.md index fc934ca93..fdb57105c 100644 --- a/docs/mkdocs/docs/api/basic_json/index.md +++ b/docs/mkdocs/docs/api/basic_json/index.md @@ -109,6 +109,9 @@ The class satisfies the following concept requirements: - **initializer_list_t** - type for initializer lists of `basic_json` values - [**input_format_t**](input_format_t.md) - type to choose the format to parse - [**json_sax_t**](../json_sax/index.md) - type for SAX events +- [**with_object_t, with_array_t, with_string_t, with_boolean_t, with_integers_t, with_float_t, with_allocator_t, + with_json_serializer_t, with_binary_t, with_base_class_t**](with_t.md) - types to create a `basic_json` type with + one (or two) replaced template parameters ### Exceptions diff --git a/docs/mkdocs/docs/api/basic_json/with_t.md b/docs/mkdocs/docs/api/basic_json/with_t.md new file mode 100644 index 000000000..2e8fee663 --- /dev/null +++ b/docs/mkdocs/docs/api/basic_json/with_t.md @@ -0,0 +1,136 @@ +# nlohmann::basic_json::with_t + +Member alias templates `with_object_t`, `with_array_t`, `with_string_t`, `with_boolean_t`, `with_integers_t`, +`with_float_t`, `with_allocator_t`, `with_json_serializer_t`, `with_binary_t`, and `with_base_class_t`. + +```cpp +template class ObjectType2> +using with_object_t = basic_json; + +template class ArrayType2> +using with_array_t = basic_json; + +template +using with_string_t = basic_json; + +template +using with_boolean_t = basic_json; + +template +using with_integers_t = basic_json; + +template +using with_float_t = basic_json; + +template class AllocatorType2> +using with_allocator_t = basic_json; + +template class JSONSerializer2> +using with_json_serializer_t = basic_json; + +template +using with_binary_t = basic_json; + +template +using with_base_class_t = basic_json; +``` + +These member alias templates make it easier to create a `basic_json` type that is identical to the current type except +for one (or, in the case of `with_integers_t`, two) of its [template parameters](index.md#template-parameters). +Spelling out all 11 template parameters of `basic_json` just to change a single one is verbose and error-prone; these +aliases only require the replacement type(s). + +with_object_t<ObjectType2> +: replaces `ObjectType` + +with_array_t<ArrayType2> +: replaces `ArrayType` + +with_string_t<StringType2> +: replaces `StringType` + +with_boolean_t<BooleanType2> +: replaces `BooleanType` + +with_integers_t<NumberIntegerType2, NumberUnsignedType2> +: replaces both `NumberIntegerType` and `NumberUnsignedType`; the two are combined into a single alias because they + are usually changed together (for instance, when switching to fixed-width integer types) + +with_float_t<NumberFloatType2> +: replaces `NumberFloatType` + +with_allocator_t<AllocatorType2> +: replaces `AllocatorType` + +with_json_serializer_t<JSONSerializer2> +: replaces `JSONSerializer` + +with_binary_t<BinaryType2> +: replaces `BinaryType` + +with_base_class_t<CustomBaseClass2> +: replaces `CustomBaseClass`; see also [`json_base_class_t`](json_base_class_t.md) + +## Notes + +All other template parameters are kept unchanged, so the resulting type still uses, for instance, the same +`ObjectType` unless `with_object_t` itself is used. + +The aliases are members of every `basic_json` specialization, including [`ordered_json`](../ordered_json.md), and the +type they produce is again a `basic_json` specialization. They can therefore be chained to replace several template +parameters at once: + +```cpp +using my_json = nlohmann::json::with_integers_t::with_float_t; +using my_ordered_json = nlohmann::ordered_json::with_string_t; +``` + +The result is the same type as spelling out all template parameters, so the order of the chained aliases does not +matter. For instance, `nlohmann::json::with_object_t` is `nlohmann::ordered_json`. + +## Examples + +??? example + + The following code shows how `with_object_t` can be used to create a JSON type that stores object elements in a + `std::map` and therefore keeps them sorted by key, unlike the default type which preserves insertion order + only when `nlohmann::ordered_json` is used. + + ```cpp + --8<-- "examples/with_t.cpp" + ``` + + Output: + + ```json + --8<-- "examples/with_t.output" + ``` + +## See also + +- [basic_json](index.md#template-parameters) - the template parameters that can be replaced +- [json_base_class_t](json_base_class_t.md) - the type used for `CustomBaseClass` + +## Version history + +- Added in version 3.13.0. diff --git a/docs/mkdocs/docs/examples/json_base_class_t.cpp b/docs/mkdocs/docs/examples/json_base_class_t.cpp index 3fb2d46a2..1dafd417f 100644 --- a/docs/mkdocs/docs/examples/json_base_class_t.cpp +++ b/docs/mkdocs/docs/examples/json_base_class_t.cpp @@ -13,19 +13,7 @@ class visitor_adaptor_with_metadata void do_visit(const Ptr& ptr, const Fnc& fnc) const; }; -using json = nlohmann::basic_json < - std::map, - std::vector, - std::string, - bool, - std::int64_t, - std::uint64_t, - double, - std::allocator, - nlohmann::adl_serializer, - std::vector, - visitor_adaptor_with_metadata - >; +using json = nlohmann::json::with_base_class_t; template void visitor_adaptor_with_metadata::visit(const Fnc& fnc) const diff --git a/docs/mkdocs/docs/examples/with_t.cpp b/docs/mkdocs/docs/examples/with_t.cpp new file mode 100644 index 000000000..665efde3e --- /dev/null +++ b/docs/mkdocs/docs/examples/with_t.cpp @@ -0,0 +1,18 @@ +#include +#include +#include + +// a JSON type that stores objects in a std::map (which keeps keys sorted) +// instead of the default ordered associative container +using sorted_json = nlohmann::json::with_object_t; + +int main() +{ + sorted_json j; + j["c"] = 1; + j["a"] = 2; + j["b"] = 3; + + // keys are sorted, because std::map is used to store the object + std::cout << j.dump() << std::endl; +} diff --git a/docs/mkdocs/docs/examples/with_t.output b/docs/mkdocs/docs/examples/with_t.output new file mode 100644 index 000000000..0886cda01 --- /dev/null +++ b/docs/mkdocs/docs/examples/with_t.output @@ -0,0 +1 @@ +{"a":2,"b":3,"c":1} diff --git a/docs/mkdocs/mkdocs.yml b/docs/mkdocs/mkdocs.yml index 5b23366bb..f38d70038 100644 --- a/docs/mkdocs/mkdocs.yml +++ b/docs/mkdocs/mkdocs.yml @@ -232,6 +232,7 @@ nav: - 'update': api/basic_json/update.md - 'value': api/basic_json/value.md - 'value_t': api/basic_json/value_t.md + - 'with_t': api/basic_json/with_t.md - byte_container_with_subtype: - 'Overview': api/byte_container_with_subtype/index.md - '(constructor)': api/byte_container_with_subtype/byte_container_with_subtype.md diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 8cf3f8397..4eeb326b8 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -226,6 +226,70 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// SAX interface type, see @ref nlohmann::json_sax using json_sax_t = json_sax; + //////////////////////////////////////////////////////////////////////////////// + // utility templates to create a json type with different template parameters // + //////////////////////////////////////////////////////////////////////////////// + + /// Json type using a different type for storing objects + /// @sa https://json.nlohmann.me/api/basic_json/with_t/ + template class ObjectType2> + using with_object_t = basic_json; + + /// Json type using a different type for storing arrays + /// @sa https://json.nlohmann.me/api/basic_json/with_t/ + template class ArrayType2> + using with_array_t = basic_json; + + /// Json type using a different type for storing strings + /// @sa https://json.nlohmann.me/api/basic_json/with_t/ + template + using with_string_t = basic_json; + + /// Json type using a different type for storing booleans + /// @sa https://json.nlohmann.me/api/basic_json/with_t/ + template + using with_boolean_t = basic_json; + + /// Json type using different types for storing signed and unsigned integers + /// @sa https://json.nlohmann.me/api/basic_json/with_t/ + template + using with_integers_t = basic_json; + + /// Json type using a different type for storing floating point numbers + /// @sa https://json.nlohmann.me/api/basic_json/with_t/ + template + using with_float_t = basic_json; + + /// Json type using a different type as base allocator + /// @sa https://json.nlohmann.me/api/basic_json/with_t/ + template class AllocatorType2> + using with_allocator_t = basic_json; + + /// Json type using a different type as json serializer + /// @sa https://json.nlohmann.me/api/basic_json/with_t/ + template class JSONSerializer2> + using with_json_serializer_t = basic_json; + + /// Json type using a different type for storing binary data + /// @sa https://json.nlohmann.me/api/basic_json/with_t/ + template + using with_binary_t = basic_json; + + /// Json type using a different type as base class + /// @sa https://json.nlohmann.me/api/basic_json/with_t/ + template + using with_base_class_t = basic_json; + //////////////// // exceptions // //////////////// diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 09e3b4f4a..be32959d5 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -27471,6 +27471,70 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// SAX interface type, see @ref nlohmann::json_sax using json_sax_t = json_sax; + //////////////////////////////////////////////////////////////////////////////// + // utility templates to create a json type with different template parameters // + //////////////////////////////////////////////////////////////////////////////// + + /// Json type using a different type for storing objects + /// @sa https://json.nlohmann.me/api/basic_json/with_t/ + template class ObjectType2> + using with_object_t = basic_json; + + /// Json type using a different type for storing arrays + /// @sa https://json.nlohmann.me/api/basic_json/with_t/ + template class ArrayType2> + using with_array_t = basic_json; + + /// Json type using a different type for storing strings + /// @sa https://json.nlohmann.me/api/basic_json/with_t/ + template + using with_string_t = basic_json; + + /// Json type using a different type for storing booleans + /// @sa https://json.nlohmann.me/api/basic_json/with_t/ + template + using with_boolean_t = basic_json; + + /// Json type using different types for storing signed and unsigned integers + /// @sa https://json.nlohmann.me/api/basic_json/with_t/ + template + using with_integers_t = basic_json; + + /// Json type using a different type for storing floating point numbers + /// @sa https://json.nlohmann.me/api/basic_json/with_t/ + template + using with_float_t = basic_json; + + /// Json type using a different type as base allocator + /// @sa https://json.nlohmann.me/api/basic_json/with_t/ + template class AllocatorType2> + using with_allocator_t = basic_json; + + /// Json type using a different type as json serializer + /// @sa https://json.nlohmann.me/api/basic_json/with_t/ + template class JSONSerializer2> + using with_json_serializer_t = basic_json; + + /// Json type using a different type for storing binary data + /// @sa https://json.nlohmann.me/api/basic_json/with_t/ + template + using with_binary_t = basic_json; + + /// Json type using a different type as base class + /// @sa https://json.nlohmann.me/api/basic_json/with_t/ + template + using with_base_class_t = basic_json; + //////////////// // exceptions // //////////////// diff --git a/tests/src/unit-allocator.cpp b/tests/src/unit-allocator.cpp index c716b3a5b..d2e23c1a7 100644 --- a/tests/src/unit-allocator.cpp +++ b/tests/src/unit-allocator.cpp @@ -48,14 +48,7 @@ TEST_CASE("bad_alloc") SECTION("bad_alloc") { // create JSON type using the throwing allocator - using bad_json = nlohmann::basic_json; + using bad_json = nlohmann::json::with_allocator_t; // creating an object should throw CHECK_THROWS_AS(bad_json(bad_json::value_t::object), std::bad_alloc&); @@ -129,14 +122,7 @@ void my_allocator_clean_up(T* p) TEST_CASE("controlled bad_alloc") { // create JSON type using the throwing allocator - using my_json = nlohmann::basic_json; + using my_json = nlohmann::json::with_allocator_t; SECTION("class json_value") { @@ -593,14 +579,7 @@ TEST_CASE("bad my_allocator::construct") { SECTION("my_allocator::construct doesn't forward") { - using bad_alloc_json = nlohmann::basic_json; + using bad_alloc_json = nlohmann::json::with_allocator_t; bad_alloc_json j; j["test"] = bad_alloc_json::array_t(); diff --git a/tests/src/unit-alt-string.cpp b/tests/src/unit-alt-string.cpp index e503a4914..961cd742b 100644 --- a/tests/src/unit-alt-string.cpp +++ b/tests/src/unit-alt-string.cpp @@ -163,16 +163,7 @@ void int_to_string(alt_string& target, std::size_t value) target = std::to_string(value).c_str(); } -using alt_json = nlohmann::basic_json < - std::map, - std::vector, - alt_string, - bool, - std::int64_t, - std::uint64_t, - double, - std::allocator, - nlohmann::adl_serializer >; +using alt_json = nlohmann::json::with_string_t; bool operator<(const char* op1, const alt_string& op2) noexcept { diff --git a/tests/src/unit-custom-base-class.cpp b/tests/src/unit-custom-base-class.cpp index b9e544705..9718d1cda 100644 --- a/tests/src/unit-custom-base-class.cpp +++ b/tests/src/unit-custom-base-class.cpp @@ -38,20 +38,7 @@ class json_metadata }; template -using json_with_metadata = - nlohmann::basic_json < - std::map, - std::vector, - std::string, - bool, - std::int64_t, - std::uint64_t, - double, - std::allocator, - nlohmann::adl_serializer, - std::vector, - json_metadata - >; +using json_with_metadata = nlohmann::json::with_base_class_t>; TEST_CASE("JSON Node Metadata") { @@ -268,19 +255,7 @@ class visitor_adaptor void do_visit(const Ptr& ptr, const Fnc& fnc) const; }; -using json_with_visitor_t = nlohmann::basic_json < - std::map, - std::vector, - std::string, - bool, - std::int64_t, - std::uint64_t, - double, - std::allocator, - nlohmann::adl_serializer, - std::vector, - visitor_adaptor - >; +using json_with_visitor_t = nlohmann::json::with_base_class_t; template void visitor_adaptor::visit(const Fnc& fnc) const diff --git a/tests/src/unit-udt.cpp b/tests/src/unit-udt.cpp index 5ad72b208..8bc04c0ba 100644 --- a/tests/src/unit-udt.cpp +++ b/tests/src/unit-udt.cpp @@ -23,6 +23,7 @@ using nlohmann::json; using namespace nlohmann::literals; // NOLINT(google-build-using-namespace) #endif +#include #include #include #include @@ -684,8 +685,7 @@ static std::ostream& operator<<(std::ostream& os, small_pod l) TEST_CASE("custom serializer for pods" * doctest::test_suite("udt")) { using custom_json = - nlohmann::basic_json; + nlohmann::json::with_json_serializer_t; auto p = udt::small_pod{42, '/', 42}; custom_json const j = p; @@ -703,7 +703,7 @@ TEST_CASE("custom serializer for pods" * doctest::test_suite("udt")) template struct another_adl_serializer; -using custom_json = nlohmann::basic_json; +using custom_json = nlohmann::json::with_json_serializer_t; template struct another_adl_serializer @@ -734,6 +734,80 @@ TEST_CASE("custom serializer that does adl by default" * doctest::test_suite("ud CHECK(me == cj.get()); } +TEST_CASE("with_*_t aliases" * doctest::test_suite("udt")) +{ + // a custom base class used to check with_base_class_t + struct custom_base_class {}; + + CHECK(std::is_same, + nlohmann::basic_json>>::value); + + CHECK(std::is_same, + nlohmann::basic_json>>::value); + + CHECK(std::is_same, + nlohmann::basic_json>>::value); + + CHECK(std::is_same, + nlohmann::basic_json>>::value); + + CHECK(std::is_same, + nlohmann::basic_json>>::value); + + CHECK(std::is_same, + nlohmann::basic_json>>::value); + + CHECK(std::is_same, + nlohmann::basic_json>>::value); + + CHECK(std::is_same, + nlohmann::basic_json>>::value); + + CHECK(std::is_same>, + nlohmann::basic_json>>::value); + + CHECK(std::is_same, + nlohmann::basic_json, custom_base_class>>::value); + + // with_string_t on ordered_json must keep ordered_map as the object type + CHECK(std::is_same, + nlohmann::basic_json>>::value); + + // the aliases are members of the resulting type, so they can be chained + CHECK(std::is_same::with_float_t, + nlohmann::basic_json>>::value); + CHECK(std::is_same::with_integers_t, + json::with_integers_t::with_float_t>::value); + + // replacing the object type of json with ordered_map yields ordered_json + CHECK(std::is_same, nlohmann::ordered_json>::value); + CHECK(std::is_same, json>::value); +} + TEST_CASE("different basic_json types conversions") { SECTION("null") From 1a4c9dea730ccba1b4a4fa006c41f1a33d5625ed Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Tue, 6 Oct 2026 22:40:10 +0200 Subject: [PATCH 07/12] Fix the standalone release tarball; update compiler list and CITATION (#5734) Squashed onto develop from: - Make the json.tar.xz release archive configure standalone - Update the supported compiler list and the CITATION.cff repository link - Add json_fwd.hpp to the Bazel singleheader-json target - Document SwiftPM's #include form and add CI coverage - Migrate REUSE metadata from deprecated .reuse/dep5 to REUSE.toml Signed-off-by: Niels Lohmann --- .github/workflows/macos.yml | 35 +++++++++++++++ .reuse/dep5 | 36 --------------- BUILD.bazel | 1 + CITATION.cff | 2 +- CMakeLists.txt | 5 ++- FILES.md | 6 +-- Makefile | 7 ++- Package.swift | 2 +- README.md | 23 +++++----- REUSE.toml | 52 ++++++++++++++++++++++ cmake/scripts/gen_bazel_build_file.cmake | 1 + cmake/scripts/gen_hedley_undef_check.cmake | 2 +- docs/mkdocs/docs/integration/cmake.md | 2 +- 13 files changed, 118 insertions(+), 56 deletions(-) delete mode 100644 .reuse/dep5 create mode 100644 REUSE.toml diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index a47fbd4b0..75afc6310 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -71,3 +71,38 @@ jobs: run: cmake --build build --parallel 10 - name: Test run: cd build ; ctest -j 10 --output-on-failure + + swiftpm: + runs-on: macos-15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Check that Package.swift resolves without a deprecation warning + run: swift package dump-package + - name: Build the SwiftPM documentation example against this checkout + run: | + mkdir -p /tmp/json-swiftpm-consumer/Sources/MyLibrary + cp docs/mkdocs/docs/integration/swift/example.cpp /tmp/json-swiftpm-consumer/Sources/MyLibrary/example.cpp + cat > /tmp/json-swiftpm-consumer/Package.swift << EOF + // swift-tools-version: 5.9 + import PackageDescription + + let package = Package( + name: "MyPackage", + dependencies: [ + .package(path: "${{ github.workspace }}") + ], + targets: [ + .target( + name: "MyLibrary", + dependencies: [ + .product(name: "json", package: "json") + ], + publicHeadersPath: "." + ) + ] + ) + EOF + cd /tmp/json-swiftpm-consumer + swift build diff --git a/.reuse/dep5 b/.reuse/dep5 deleted file mode 100644 index d12cb816c..000000000 --- a/.reuse/dep5 +++ /dev/null @@ -1,36 +0,0 @@ -Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ -Upstream-Name: json -Upstream-Contact: Niels Lohmann -Source: https://github.com/nlohmann/json - -Files: * -Copyright: 2013-2026 Niels Lohmann -License: MIT - -Files: include/nlohmann/thirdparty/hedley.hpp -Copyright: 2016-2021 Evan Nemerson -License: CC0 - -Files: include/nlohmann/detail/meta/cpp_future.hpp -Copyright: 2013-2026 Niels Lohmann and 2018 The Abseil Authors -License: MIT AND Apache-2.0 - -Files: tests/thirdparty/doctest/* -Copyright: 2016-2023 Viktor Kirilov -License: MIT - -Files: tests/thirdparty/fifo_map/* -Copyright: 2015-2017 Niels Lohmann -License: MIT - -Files: tests/thirdparty/imapdl/* -Copyright: 2017 Georg Sauthoff -License: GPL-3.0-only - -Files: tools/amalgamate/* -Copyright: 2012 Erik Edlund -License: BSD-3-Clause - -Files: tools/gdb_pretty_printer/* -Copyright: 2020 Hannes Domani -License: MIT diff --git a/BUILD.bazel b/BUILD.bazel index 3dfffad1e..268c3e959 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -80,6 +80,7 @@ cc_library( name = "singleheader-json", hdrs = [ "single_include/nlohmann/json.hpp", + "single_include/nlohmann/json_fwd.hpp", ], includes = ["single_include"], visibility = ["//visibility:public"], diff --git a/CITATION.cff b/CITATION.cff index 8a70c17bc..eb5118999 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -10,5 +10,5 @@ title: "JSON for Modern C++" version: 3.12.0 date-released: 2025-04-07 license: MIT -repository-code: "https://github.com/nlohmann" +repository-code: "https://github.com/nlohmann/json" url: https://json.nlohmann.me diff --git a/CMakeLists.txt b/CMakeLists.txt index a2501a92d..fed9e3171 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,8 +42,11 @@ endif() ## OPTIONS ## +# Build the tests by default only for the main project and only if the tests +# directory exists (the release archive json.tar.xz does not contain it). # VERSION_GREATER_EQUAL is not available in older CMake (< 3.7) -if(${MAIN_PROJECT} AND (${CMAKE_VERSION} VERSION_EQUAL 3.13 OR ${CMAKE_VERSION} VERSION_GREATER 3.13)) +if(${MAIN_PROJECT} AND (${CMAKE_VERSION} VERSION_EQUAL 3.13 OR ${CMAKE_VERSION} VERSION_GREATER 3.13) + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests/CMakeLists.txt") set(JSON_BuildTests_INIT ON) else() set(JSON_BuildTests_INIT OFF) diff --git a/FILES.md b/FILES.md index d13be1238..861fdf610 100644 --- a/FILES.md +++ b/FILES.md @@ -196,19 +196,19 @@ Further documentation: ## REUSE -### `.reuse/dep5` +### `REUSE.toml` The file defines the licenses of certain third-party components in the repository. The root `Makefile` contains a target `reuse` that checks for compliance. Further documentation: -- [DEP5](https://reuse.software/spec-3.2/#dep5-deprecated) +- [REUSE.toml](https://reuse.software/spec-3.3/#reusetoml) - [reuse command-line tool](https://pypi.org/project/reuse/) - [documentation of linting](https://reuse.readthedocs.io/en/stable/man/reuse-lint.html) - [REUSE](http://reuse.software) > [!IMPORTANT] -> The filename `.reuse/dep5` is predetermined by REUSE. Alternatively, a `REUSE.toml` file can be used. +> The filename `REUSE.toml` is predetermined by REUSE. Alternatively, a `.reuse/dep5` file (deprecated) can be used. ### `.reuse/templates` diff --git a/Makefile b/Makefile index 3a7fa5c41..11b9227a4 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,9 @@ # find GNU sed to use `-i` parameter SED:=$(shell command -v gsed || which sed) +# find GNU tar to use `--sort` and `--pax-option` parameters +TAR:=$(shell command -v gtar || which tar) + ########################################################################## # source files @@ -174,8 +177,8 @@ ChangeLog.md: # archive is created according to the advices of . json.tar.xz: mkdir json - rsync -R $(shell find LICENSE.MIT nlohmann_json.natvis CMakeLists.txt cmake/*.in include single_include -type f) json - gtar --sort=name --mtime="@$(shell git log -1 --pretty=%ct)" --owner=0 --group=0 --numeric-owner --pax-option=exthdr.name=%d/PaxHeaders/%f,delete=atime,delete=ctime --create --file - json | xz --compress -9e --threads=2 - > json.tar.xz + rsync -R $(shell find LICENSE.MIT nlohmann_json.natvis CMakeLists.txt cmake/*.in include single_include src/modules -type f) json + $(TAR) --sort=name --mtime="@$(shell git log -1 --pretty=%ct)" --owner=0 --group=0 --numeric-owner --pax-option=exthdr.name=%d/PaxHeaders/%f,delete=atime,delete=ctime --create --file - json | xz --compress -9e --threads=2 - > json.tar.xz rm -fr json # We use `-X` to make the resulting ZIP file reproducible, see diff --git a/Package.swift b/Package.swift index 2f1e654a2..09b12dea3 100644 --- a/Package.swift +++ b/Package.swift @@ -6,7 +6,7 @@ import PackageDescription let package = Package( name: "nlohmann-json", platforms: [ - .iOS(.v12), .macOS(.v10_13), .tvOS(.v12), .watchOS(.v4), .visionOS(.v1) + .iOS(.v12), .macOS(.v10_13), .tvOS(.v12), .watchOS(.v9), .visionOS(.v1) ], products: [ .library(name: "json", targets: ["json"]) diff --git a/README.md b/README.md index ae0ad12d2..18665dbf8 100644 --- a/README.md +++ b/README.md @@ -1202,15 +1202,18 @@ language bindings, format converters, and the like. See the curated [Ecosystem]( Though it's 2026 already, the support for C++11 is still a bit sparse. Currently, the following compilers are known to work: -- GCC 4.8 - 14.2 (and possibly later) -- Clang 3.4 - 21.0 (and possibly later) -- Apple Clang 9.1 - 16.0 (and possibly later) -- Intel C++ Compiler 17.0.2 (and possibly later) -- Nvidia CUDA Compiler 11.0.221 (and possibly later) -- Microsoft Visual C++ 2015 / Build Tools 14.0.25123.0 (and possibly later) -- Microsoft Visual C++ 2017 / Build Tools 15.5.180.51428 (and possibly later) -- Microsoft Visual C++ 2019 / Build Tools 16.3.1+1def00d3d (and possibly later) -- Microsoft Visual C++ 2022 / Build Tools 19.30.30709.0 (and possibly later) +- GCC 4.8 - 16.2 (and possibly later) +- Clang 3.4 - 22.1 (and possibly later) +- Apple Clang 15.0 - 21.0 (and possibly later) +- Intel C++ Compiler Classic (icpc) 2021.10 +- Intel oneAPI DPC++/C++ Compiler (icpx) 2025.3 (and possibly later) +- NVIDIA CUDA Compiler (nvcc) 11.8 - 12.6 (and possibly later) +- NVIDIA HPC SDK C++ Compiler (nvc++) 25.5 (and possibly later) +- Microsoft Visual C++ 2015 / MSVC 19.0 (and possibly later) +- Microsoft Visual C++ 2017 / MSVC 19.16 (and possibly later) +- Microsoft Visual C++ 2019 / MSVC 19.29 (and possibly later) +- Microsoft Visual C++ 2022 / MSVC 19.44 (and possibly later) +- Microsoft Visual C++ 2026 / MSVC 19.51 (and possibly later) I would be happy to learn about other compilers/versions. @@ -1401,7 +1404,7 @@ The library is compliant to version 3.3 of the [**REUSE specification**](https:/ - Every source file contains an SPDX copyright header. - The full text of all licenses used in the repository can be found in the `LICENSES` folder. -- File `.reuse/dep5` contains an overview of all files' copyrights and licenses. +- File `REUSE.toml` contains an overview of all files' copyrights and licenses. - Run `pipx run reuse lint` to verify the project's REUSE compliance and `pipx run reuse spdx` to generate a SPDX SBOM. ## Contact diff --git a/REUSE.toml b/REUSE.toml new file mode 100644 index 000000000..47bcab16a --- /dev/null +++ b/REUSE.toml @@ -0,0 +1,52 @@ +version = 1 +SPDX-PackageName = "json" +SPDX-PackageSupplier = "Niels Lohmann " +SPDX-PackageDownloadLocation = "https://github.com/nlohmann/json" + +[[annotations]] +path = "**" +precedence = "aggregate" +SPDX-FileCopyrightText = "2013-2026 Niels Lohmann " +SPDX-License-Identifier = "MIT" + +[[annotations]] +path = "include/nlohmann/thirdparty/hedley.hpp" +precedence = "aggregate" +SPDX-FileCopyrightText = "2016-2021 Evan Nemerson " +SPDX-License-Identifier = "CC0" + +[[annotations]] +path = "include/nlohmann/detail/meta/cpp_future.hpp" +precedence = "aggregate" +SPDX-FileCopyrightText = "2013-2026 Niels Lohmann and 2018 The Abseil Authors" +SPDX-License-Identifier = "MIT AND Apache-2.0" + +[[annotations]] +path = "tests/thirdparty/doctest/**" +precedence = "aggregate" +SPDX-FileCopyrightText = "2016-2023 Viktor Kirilov" +SPDX-License-Identifier = "MIT" + +[[annotations]] +path = "tests/thirdparty/fifo_map/**" +precedence = "aggregate" +SPDX-FileCopyrightText = "2015-2017 Niels Lohmann" +SPDX-License-Identifier = "MIT" + +[[annotations]] +path = "tests/thirdparty/imapdl/**" +precedence = "aggregate" +SPDX-FileCopyrightText = "2017 Georg Sauthoff " +SPDX-License-Identifier = "GPL-3.0-only" + +[[annotations]] +path = "tools/amalgamate/**" +precedence = "aggregate" +SPDX-FileCopyrightText = "2012 Erik Edlund " +SPDX-License-Identifier = "BSD-3-Clause" + +[[annotations]] +path = "tools/gdb_pretty_printer/**" +precedence = "aggregate" +SPDX-FileCopyrightText = "2020 Hannes Domani " +SPDX-License-Identifier = "MIT" diff --git a/cmake/scripts/gen_bazel_build_file.cmake b/cmake/scripts/gen_bazel_build_file.cmake index a781a2e3f..ca5069cb0 100644 --- a/cmake/scripts/gen_bazel_build_file.cmake +++ b/cmake/scripts/gen_bazel_build_file.cmake @@ -48,6 +48,7 @@ cc_library( name = "singleheader-json", hdrs = [ "single_include/nlohmann/json.hpp", + "single_include/nlohmann/json_fwd.hpp", ], includes = ["single_include"], visibility = ["//visibility:public"], diff --git a/cmake/scripts/gen_hedley_undef_check.cmake b/cmake/scripts/gen_hedley_undef_check.cmake index fc8cfec2c..f796588c6 100644 --- a/cmake/scripts/gen_hedley_undef_check.cmake +++ b/cmake/scripts/gen_hedley_undef_check.cmake @@ -64,7 +64,7 @@ if(MODE STREQUAL "undef") # recipe is self-contained and its output is byte-stable across reruns. # The embedded SPDX tags below are part of the *generated* file's # content, not a REUSE header for this .cmake script itself (which is - # already covered by the blanket "Files: *" rule in .reuse/dep5) -- keep + # already covered by the blanket path = "**" rule in REUSE.toml) -- keep # them wrapped in REUSE-IgnoreStart/End so `reuse lint` does not try to # parse "MIT\n")" as this file's own SPDX-License-Identifier value. # REUSE-IgnoreStart diff --git a/docs/mkdocs/docs/integration/cmake.md b/docs/mkdocs/docs/integration/cmake.md index d75aef47b..9492efe9f 100644 --- a/docs/mkdocs/docs/integration/cmake.md +++ b/docs/mkdocs/docs/integration/cmake.md @@ -126,7 +126,7 @@ automatically download a release as a dependency at configure time. ### `JSON_BuildTests` -Build the unit tests when [`BUILD_TESTING`](https://cmake.org/cmake/help/latest/command/enable_testing.html) is enabled. This option is `ON` by default if the library's CMake project is the top project. That is, when integrating the library as described above, the test suite is not built unless explicitly switched on with this option. +Build the unit tests when [`BUILD_TESTING`](https://cmake.org/cmake/help/latest/command/enable_testing.html) is enabled. This option is `ON` by default if the library's CMake project is the top project and the `tests` directory exists (the release archive `json.tar.xz` does not contain it). That is, when integrating the library as described above, the test suite is not built unless explicitly switched on with this option. ### `JSON_CI` From 0c2329d0e2f4dec7a3e215ffe220540e7ae71e5f Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Tue, 6 Oct 2026 22:53:03 +0200 Subject: [PATCH 08/12] Reduce test suite runtime and run the Unicode tests everywhere (#5605) Squashed onto develop from: - Cut Unicode ill-formed byte sweeps to one representative prefix - Pin unrelated bytes in the remaining ill-formed UTF-8 sweeps - Speed up unit-unicode1 - Run the cheap binary format size tests unconditionally - Compile unit-msgpack.cpp only once - Check the JSON Pointer roundtrip for every code point again - Cover every byte class in the ill-formed UTF-8 sweeps - Merge the Unicode tests into unit-unicode.cpp - Stop excluding the Unicode tests in CI Signed-off-by: elix3r <157088510+22elix3r@users.noreply.github.com> Signed-off-by: Niels Lohmann Co-authored-by: elix3r <157088510+22elix3r@users.noreply.github.com> --- .github/external_ci/appveyor.yml | 7 +- .github/workflows/windows.yml | 4 +- cmake/ci.cmake | 13 +- tests/CMakeLists.txt | 3 - tests/src/test_utils.hpp | 28 + tests/src/unit-binary_formats.cpp | 88 +- tests/src/unit-msgpack-cpp17.cpp | 99 ++ tests/src/unit-msgpack.cpp | 79 -- tests/src/unit-unicode.cpp | 1609 +++++++++++++++++++++++++++++ tests/src/unit-unicode1.cpp | 623 ----------- tests/src/unit-unicode2.cpp | 612 ----------- tests/src/unit-unicode3.cpp | 326 ------ tests/src/unit-unicode4.cpp | 326 ------ tests/src/unit-unicode5.cpp | 326 ------ 14 files changed, 1791 insertions(+), 2352 deletions(-) create mode 100644 tests/src/unit-msgpack-cpp17.cpp create mode 100644 tests/src/unit-unicode.cpp delete mode 100644 tests/src/unit-unicode1.cpp delete mode 100644 tests/src/unit-unicode2.cpp delete mode 100644 tests/src/unit-unicode3.cpp delete mode 100644 tests/src/unit-unicode4.cpp delete mode 100644 tests/src/unit-unicode5.cpp diff --git a/.github/external_ci/appveyor.yml b/.github/external_ci/appveyor.yml index 4f0d3f5e3..21bb2e27a 100644 --- a/.github/external_ci/appveyor.yml +++ b/.github/external_ci/appveyor.yml @@ -87,9 +87,4 @@ build_script: - cmake --build . --config "%configuration%" --parallel 2 test_script: - - if "%configuration%"=="Release" ctest -C "%configuration%" --parallel 2 --output-on-failure - # On Debug builds, skip test-unicode_all - # as it is extremely slow to run and cause - # occasional timeouts on AppVeyor. - # More info: https://github.com/nlohmann/json/pull/1570 - - if "%configuration%"=="Debug" ctest --exclude-regex "test-unicode" -C "%configuration%" --parallel 2 --output-on-failure + - ctest -C "%configuration%" --parallel 2 --output-on-failure diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 65169eac4..855d8af95 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -178,7 +178,7 @@ jobs: - name: Build run: cmake --build build --parallel 10 - name: Test - run: cd build ; ctest -j 10 -C Debug --exclude-regex "test-unicode" --output-on-failure + run: cd build ; ctest -j 10 -C Debug --output-on-failure clang-cl-12: runs-on: windows-2022 @@ -195,7 +195,7 @@ jobs: - name: Build run: cmake --build build --config Debug --parallel 10 - name: Test - run: cd build ; ctest -j 10 -C Debug --exclude-regex "test-unicode" --output-on-failure + run: cd build ; ctest -j 10 -C Debug --output-on-failure ci_module_cpp20: runs-on: windows-2022 diff --git a/cmake/ci.cmake b/cmake/ci.cmake index c4bced5ba..15345c5d6 100644 --- a/cmake/ci.cmake +++ b/cmake/ci.cmake @@ -456,13 +456,14 @@ add_custom_target(ci_test_single_header # Valgrind. ############################################################################### +# The Unicode test (~17M assertions) is too slow under Valgrind. add_custom_target(ci_test_valgrind COMMAND CXX=${GCC_TOOL} ${CMAKE_COMMAND} -DCMAKE_BUILD_TYPE=Debug -GNinja -DJSON_BuildTests=ON -DJSON_Valgrind=ON -S${PROJECT_SOURCE_DIR} -B${PROJECT_BINARY_DIR}/build_valgrind COMMAND ${CMAKE_COMMAND} --build ${PROJECT_BINARY_DIR}/build_valgrind - COMMAND cd ${PROJECT_BINARY_DIR}/build_valgrind && ${CMAKE_CTEST_COMMAND} -L valgrind --parallel ${N} --output-on-failure + COMMAND cd ${PROJECT_BINARY_DIR}/build_valgrind && ${CMAKE_CTEST_COMMAND} -L valgrind --exclude-regex "test-unicode" --parallel ${N} --output-on-failure COMMENT "Compile and test with Valgrind" ) @@ -787,7 +788,7 @@ foreach(COMPILER g++-4.8 g++-4.9 g++-5 g++-6 g++-7 g++-8 g++-9 g++-10 g++-11 cla -S${PROJECT_SOURCE_DIR} -B${PROJECT_BINARY_DIR}/build_compiler_${COMPILER} ${ADDITIONAL_FLAGS} COMMAND ${CMAKE_COMMAND} --build ${PROJECT_BINARY_DIR}/build_compiler_${COMPILER} - COMMAND cd ${PROJECT_BINARY_DIR}/build_compiler_${COMPILER} && ${CMAKE_CTEST_COMMAND} --parallel ${N} --exclude-regex "test-unicode" --output-on-failure + COMMAND cd ${PROJECT_BINARY_DIR}/build_compiler_${COMPILER} && ${CMAKE_CTEST_COMMAND} --parallel ${N} --output-on-failure COMMENT "Compile and test with ${COMPILER}" ) endif() @@ -801,7 +802,7 @@ add_custom_target(ci_test_compiler_default -S${PROJECT_SOURCE_DIR} -B${PROJECT_BINARY_DIR}/build_compiler_default ${ADDITIONAL_FLAGS} COMMAND ${CMAKE_COMMAND} --build ${PROJECT_BINARY_DIR}/build_compiler_default --parallel ${N} - COMMAND cd ${PROJECT_BINARY_DIR}/build_compiler_default && ${CMAKE_CTEST_COMMAND} --parallel ${N} --exclude-regex "test-unicode" -LE git_required --output-on-failure + COMMAND cd ${PROJECT_BINARY_DIR}/build_compiler_default && ${CMAKE_CTEST_COMMAND} --parallel ${N} -LE git_required --output-on-failure COMMENT "Compile and test with default C++ compiler" ) @@ -839,7 +840,7 @@ add_custom_target(ci_icpc -DJSON_BuildTests=ON -DJSON_FastTests=ON -S${PROJECT_SOURCE_DIR} -B${PROJECT_BINARY_DIR}/build_icpc COMMAND ${CMAKE_COMMAND} --build ${PROJECT_BINARY_DIR}/build_icpc - COMMAND cd ${PROJECT_BINARY_DIR}/build_icpc && ${CMAKE_CTEST_COMMAND} --parallel ${N} --exclude-regex "test-unicode" --output-on-failure + COMMAND cd ${PROJECT_BINARY_DIR}/build_icpc && ${CMAKE_CTEST_COMMAND} --parallel ${N} --output-on-failure COMMENT "Compile and test with ICPC" ) @@ -850,7 +851,7 @@ add_custom_target(ci_icpx -DJSON_BuildTests=ON -DJSON_FastTests=ON -S${PROJECT_SOURCE_DIR} -B${PROJECT_BINARY_DIR}/build_icpx COMMAND ${CMAKE_COMMAND} --build ${PROJECT_BINARY_DIR}/build_icpx - COMMAND cd ${PROJECT_BINARY_DIR}/build_icpx && ${CMAKE_CTEST_COMMAND} --parallel ${N} --exclude-regex "test-unicode" --output-on-failure + COMMAND cd ${PROJECT_BINARY_DIR}/build_icpx && ${CMAKE_CTEST_COMMAND} --parallel ${N} --output-on-failure COMMENT "Compile and test with ICPX (Intel oneAPI DPC++/C++)" ) @@ -886,7 +887,7 @@ add_custom_target(ci_nvhpc COMMAND ${CMAKE_COMMAND} --build ${PROJECT_BINARY_DIR}/build_nvhpc # the pipes are escaped so the surrounding shell passes them to ctest verbatim # instead of treating them as shell pipe operators - COMMAND cd ${PROJECT_BINARY_DIR}/build_nvhpc && ${CMAKE_CTEST_COMMAND} --parallel ${N} --exclude-regex "test-unicode\\|test-comparison_cpp20\\|test-comparison_legacy_cpp20\\|test-constructor1_cpp11\\|test-deserialization_cpp20" --output-on-failure + COMMAND cd ${PROJECT_BINARY_DIR}/build_nvhpc && ${CMAKE_CTEST_COMMAND} --parallel ${N} --exclude-regex "test-comparison_cpp20\\|test-comparison_legacy_cpp20\\|test-constructor1_cpp11\\|test-deserialization_cpp20" --output-on-failure COMMENT "Compile and test with NVIDIA HPC SDK (nvc++)" ) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 8735934de..f98ab7593 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -135,9 +135,6 @@ json_test_set_test_options(test-disabled_exceptions #$<$:/EH> ) -# raise timeout of expensive Unicode test -json_test_set_test_options(test-unicode4 TEST_PROPERTIES TIMEOUT 3000) - # only the #972 regression test needs thirdparty/fifo_map on its include path json_test_set_test_options(test-regression1 LINK_LIBRARIES fifo_map_include) diff --git a/tests/src/test_utils.hpp b/tests/src/test_utils.hpp index 8f7bd06a0..58f1f5409 100644 --- a/tests/src/test_utils.hpp +++ b/tests/src/test_utils.hpp @@ -8,6 +8,7 @@ #pragma once +#include // array #include // uint8_t #include // size_t #include // ifstream, ios @@ -43,6 +44,33 @@ T next_integer_sample(T i, T last, T stride) return n < last ? n : last; } +// UTF-8 continuation bytes in [lo, hi] that stand in for all of them in the +// ill-formed UTF-8 tests. Both the lexer's range checks and the serializer's +// decoder (detail::decode) only distinguish the classes 0x80..0x8F, 0x90..0x9F, +// and 0xA0..0xBF, so the first and last byte of each class within [lo, hi] +// exercise every behavior while a test sweeps another byte position through +// all 256 values (#5418). Define JSON_TEST_UTF8_EXHAUSTIVE to get every byte. +inline std::vector utf8_continuation_bytes(int lo, int hi) +{ + std::vector result; +#ifdef JSON_TEST_UTF8_EXHAUSTIVE + for (int byte = lo; byte <= hi; ++byte) + { + result.push_back(byte); + } +#else + static const std::array class_ends = {{0x80, 0x8F, 0x90, 0x9F, 0xA0, 0xBF}}; + for (const int byte : class_ends) + { + if (lo <= byte && byte <= hi) + { + result.push_back(byte); + } + } +#endif + return result; +} + inline std::vector read_binary_file(const std::string& filename) { std::ifstream file(filename, std::ios::binary); diff --git a/tests/src/unit-binary_formats.cpp b/tests/src/unit-binary_formats.cpp index 846a092fb..3e5edf765 100644 --- a/tests/src/unit-binary_formats.cpp +++ b/tests/src/unit-binary_formats.cpp @@ -19,7 +19,7 @@ using nlohmann::json; #include #include "make_test_data_available.hpp" -TEST_CASE("Binary Formats" * doctest::skip()) +TEST_CASE("Binary Formats") { SECTION("canada.json") { @@ -147,48 +147,6 @@ TEST_CASE("Binary Formats" * doctest::skip()) CHECK((100.0 * double(ubjson_3_size) / double(json_size)) == Approx(84.963)); } - SECTION("jeopardy.json") - { - const auto* filename = TEST_DATA_DIRECTORY "/jeopardy/jeopardy.json"; - json j = json::parse(std::ifstream(filename)); - - const auto json_size = j.dump().size(); - const auto bjdata_1_size = json::to_bjdata(j).size(); - const auto bjdata_2_size = json::to_bjdata(j, true).size(); - const auto bjdata_3_size = json::to_bjdata(j, true, true).size(); - const auto bon8_size = json::to_bon8(j).size(); - const auto bson_size = json::to_bson({{"", j}}).size(); // wrap array in object for BSON - const auto cbor_size = json::to_cbor(j).size(); - const auto msgpack_size = json::to_msgpack(j).size(); - const auto ubjson_1_size = json::to_ubjson(j).size(); - const auto ubjson_2_size = json::to_ubjson(j, true).size(); - const auto ubjson_3_size = json::to_ubjson(j, true, true).size(); - - CHECK(json_size == 52508728); - CHECK(bjdata_1_size == 50710965); - CHECK(bjdata_2_size == 51144830); - CHECK(bjdata_3_size == 51144830); - CHECK(bon8_size == 45942080); - CHECK(bson_size == 56008520); - CHECK(cbor_size == 46187320); - CHECK(msgpack_size == 46158575); - CHECK(ubjson_1_size == 50710965); - CHECK(ubjson_2_size == 51144830); - CHECK(ubjson_3_size == 49861422); - - CHECK((100.0 * double(json_size) / double(json_size)) == Approx(100.0)); - CHECK((100.0 * double(bjdata_1_size) / double(json_size)) == Approx(96.576)); - CHECK((100.0 * double(bjdata_2_size) / double(json_size)) == Approx(97.402)); - CHECK((100.0 * double(bjdata_3_size) / double(json_size)) == Approx(97.402)); - CHECK((100.0 * double(bon8_size) / double(json_size)) == Approx(87.494)); - CHECK((100.0 * double(bson_size) / double(json_size)) == Approx(106.665)); - CHECK((100.0 * double(cbor_size) / double(json_size)) == Approx(87.961)); - CHECK((100.0 * double(msgpack_size) / double(json_size)) == Approx(87.906)); - CHECK((100.0 * double(ubjson_1_size) / double(json_size)) == Approx(96.576)); - CHECK((100.0 * double(ubjson_2_size) / double(json_size)) == Approx(97.402)); - CHECK((100.0 * double(ubjson_3_size) / double(json_size)) == Approx(94.958)); - } - SECTION("sample.json") { const auto* filename = TEST_DATA_DIRECTORY "/json_testsuite/sample.json"; @@ -230,6 +188,50 @@ TEST_CASE("Binary Formats" * doctest::skip()) } } +// jeopardy.json is 52 MB and produces ~500 MB of serialization output, so it +// is kept apart from the cheap corpus files above (#5418) +TEST_CASE("Binary Formats (jeopardy.json)" * doctest::skip()) +{ + const auto* filename = TEST_DATA_DIRECTORY "/jeopardy/jeopardy.json"; + json j = json::parse(std::ifstream(filename)); + + const auto json_size = j.dump().size(); + const auto bjdata_1_size = json::to_bjdata(j).size(); + const auto bjdata_2_size = json::to_bjdata(j, true).size(); + const auto bjdata_3_size = json::to_bjdata(j, true, true).size(); + const auto bon8_size = json::to_bon8(j).size(); + const auto bson_size = json::to_bson({{"", j}}).size(); // wrap array in object for BSON + const auto cbor_size = json::to_cbor(j).size(); + const auto msgpack_size = json::to_msgpack(j).size(); + const auto ubjson_1_size = json::to_ubjson(j).size(); + const auto ubjson_2_size = json::to_ubjson(j, true).size(); + const auto ubjson_3_size = json::to_ubjson(j, true, true).size(); + + CHECK(json_size == 52508728); + CHECK(bjdata_1_size == 50710965); + CHECK(bjdata_2_size == 51144830); + CHECK(bjdata_3_size == 51144830); + CHECK(bon8_size == 45942080); + CHECK(bson_size == 56008520); + CHECK(cbor_size == 46187320); + CHECK(msgpack_size == 46158575); + CHECK(ubjson_1_size == 50710965); + CHECK(ubjson_2_size == 51144830); + CHECK(ubjson_3_size == 49861422); + + CHECK((100.0 * double(json_size) / double(json_size)) == Approx(100.0)); + CHECK((100.0 * double(bjdata_1_size) / double(json_size)) == Approx(96.576)); + CHECK((100.0 * double(bjdata_2_size) / double(json_size)) == Approx(97.402)); + CHECK((100.0 * double(bjdata_3_size) / double(json_size)) == Approx(97.402)); + CHECK((100.0 * double(bon8_size) / double(json_size)) == Approx(87.494)); + CHECK((100.0 * double(bson_size) / double(json_size)) == Approx(106.665)); + CHECK((100.0 * double(cbor_size) / double(json_size)) == Approx(87.961)); + CHECK((100.0 * double(msgpack_size) / double(json_size)) == Approx(87.906)); + CHECK((100.0 * double(ubjson_1_size) / double(json_size)) == Approx(96.576)); + CHECK((100.0 * double(ubjson_2_size) / double(json_size)) == Approx(97.402)); + CHECK((100.0 * double(ubjson_3_size) / double(json_size)) == Approx(94.958)); +} + namespace { // the binary formats as function pointers for "Binary formats with narrow number types"; diff --git a/tests/src/unit-msgpack-cpp17.cpp b/tests/src/unit-msgpack-cpp17.cpp new file mode 100644 index 000000000..6ae4d6d56 --- /dev/null +++ b/tests/src/unit-msgpack-cpp17.cpp @@ -0,0 +1,99 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +// This file contains the C++17-only part of unit-msgpack.cpp (std::byte +// input). It is kept in a separate translation unit so the (much larger) +// unit-msgpack.cpp does not need to be compiled and run a second time just +// for this one test case (#5418). + +#include "doctest_compatibility.h" + +#include +using nlohmann::json; + +#ifdef JSON_HAS_CPP_17 +#include +#include + +// Test suite for verifying MessagePack handling with std::byte input +TEST_CASE("MessagePack with std::byte") +{ + + SECTION("std::byte compatibility") + { + SECTION("vector roundtrip") + { + json original = + { + {"name", "test"}, + {"value", 42}, + {"array", {1, 2, 3}} + }; + + std::vector temp = json::to_msgpack(original); + // Convert the uint8_t vector to std::byte vector + std::vector msgpack_data(temp.size()); + for (size_t i = 0; i < temp.size(); ++i) + { + msgpack_data[i] = std::byte(temp[i]); + } + // Deserialize from std::byte vector back to JSON + json from_bytes; + CHECK_NOTHROW(from_bytes = json::from_msgpack(msgpack_data)); + + CHECK(from_bytes == original); + } + + SECTION("empty vector") + { + const std::vector empty_data; + CHECK_THROWS_WITH_AS([&]() + { + [[maybe_unused]] auto result = json::from_msgpack(empty_data); + return true; + } + (), + "[json.exception.parse_error.110] parse error at byte 1: syntax error while parsing MessagePack value: unexpected end of input", + json::parse_error&); + } + + SECTION("comparison with workaround") + { + json original = + { + {"string", "hello"}, + {"integer", 42}, + {"float", 3.14}, + {"boolean", true}, + {"null", nullptr}, + {"array", {1, 2, 3}}, + {"object", {{"key", "value"}}} + }; + + std::vector temp = json::to_msgpack(original); + + std::vector msgpack_data(temp.size()); + for (size_t i = 0; i < temp.size(); ++i) + { + msgpack_data[i] = std::byte(temp[i]); + } + // Attempt direct deserialization using std::byte input + const json direct_result = json::from_msgpack(msgpack_data); + + // Test the workaround approach: reinterpret as unsigned char* and use iterator range + const auto* const char_start = reinterpret_cast(msgpack_data.data()); + const auto* const char_end = char_start + msgpack_data.size(); + json workaround_result = json::from_msgpack(char_start, char_end); + + // Verify that the final deserialized JSON matches the original JSON + CHECK(direct_result == workaround_result); + CHECK(direct_result == original); + } + } +} +#endif diff --git a/tests/src/unit-msgpack.cpp b/tests/src/unit-msgpack.cpp index bc29b301b..47cccbf13 100644 --- a/tests/src/unit-msgpack.cpp +++ b/tests/src/unit-msgpack.cpp @@ -2173,85 +2173,6 @@ TEST_CASE("MessagePack roundtrips" * doctest::skip()) } } -#ifdef JSON_HAS_CPP_17 -// Test suite for verifying MessagePack handling with std::byte input -TEST_CASE("MessagePack with std::byte") -{ - - SECTION("std::byte compatibility") - { - SECTION("vector roundtrip") - { - json original = - { - {"name", "test"}, - {"value", 42}, - {"array", {1, 2, 3}} - }; - - std::vector temp = json::to_msgpack(original); - // Convert the uint8_t vector to std::byte vector - std::vector msgpack_data(temp.size()); - for (size_t i = 0; i < temp.size(); ++i) - { - msgpack_data[i] = std::byte(temp[i]); - } - // Deserialize from std::byte vector back to JSON - json from_bytes; - CHECK_NOTHROW(from_bytes = json::from_msgpack(msgpack_data)); - - CHECK(from_bytes == original); - } - - SECTION("empty vector") - { - const std::vector empty_data; - CHECK_THROWS_WITH_AS([&]() - { - [[maybe_unused]] auto result = json::from_msgpack(empty_data); - return true; - } - (), - "[json.exception.parse_error.110] parse error at byte 1: syntax error while parsing MessagePack value: unexpected end of input", - json::parse_error&); - } - - SECTION("comparison with workaround") - { - json original = - { - {"string", "hello"}, - {"integer", 42}, - {"float", 3.14}, - {"boolean", true}, - {"null", nullptr}, - {"array", {1, 2, 3}}, - {"object", {{"key", "value"}}} - }; - - std::vector temp = json::to_msgpack(original); - - std::vector msgpack_data(temp.size()); - for (size_t i = 0; i < temp.size(); ++i) - { - msgpack_data[i] = std::byte(temp[i]); - } - // Attempt direct deserialization using std::byte input - const json direct_result = json::from_msgpack(msgpack_data); - - // Test the workaround approach: reinterpret as unsigned char* and use iterator range - const auto* const char_start = reinterpret_cast(msgpack_data.data()); - const auto* const char_end = char_start + msgpack_data.size(); - json workaround_result = json::from_msgpack(char_start, char_end); - - // Verify that the final deserialized JSON matches the original JSON - CHECK(direct_result == workaround_result); - CHECK(direct_result == original); - } - } -} -#endif - // the fake sizes below do not fit into a 32-bit std::size_t // with clang and libstdc++ 10, the std::filesystem::path conversion that // C++17 builds consider for every string type is ambiguous for a class diff --git a/tests/src/unit-unicode.cpp b/tests/src/unit-unicode.cpp new file mode 100644 index 000000000..97949ed2d --- /dev/null +++ b/tests/src/unit-unicode.cpp @@ -0,0 +1,1609 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#include "doctest_compatibility.h" + +#include +using nlohmann::json; + +#include +#include +#include "make_test_data_available.hpp" +#include "test_utils.hpp" + +// this test suite uses static variables with non-trivial destructors +DOCTEST_CLANG_SUPPRESS_WARNING_PUSH +DOCTEST_CLANG_SUPPRESS_WARNING("-Wexit-time-destructors") + +namespace +{ +void check_utf8dump(bool success_expected, int byte1, int byte2, int byte3, int byte4); + +void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3 = -1, int byte4 = -1) +{ + static std::string json_string; + json_string.clear(); + + CAPTURE(byte1) + CAPTURE(byte2) + CAPTURE(byte3) + CAPTURE(byte4) + + json_string += std::string(1, static_cast(byte1)); + + if (byte2 != -1) + { + json_string += std::string(1, static_cast(byte2)); + } + + if (byte3 != -1) + { + json_string += std::string(1, static_cast(byte3)); + } + + if (byte4 != -1) + { + json_string += std::string(1, static_cast(byte4)); + } + + CAPTURE(json_string) + + // store the string in a JSON value + static json j; + static json j2; + j = json_string; + j2 = "abc" + json_string + "xyz"; + + static std::string s_ignored; + static std::string s_ignored2; + static std::string s_ignored_ascii; + static std::string s_ignored2_ascii; + static std::string s_replaced; + static std::string s_replaced2; + static std::string s_replaced_ascii; + static std::string s_replaced2_ascii; + + // dumping with ignore/replace must not throw in any case + s_ignored = j.dump(-1, ' ', false, json::error_handler_t::ignore); + s_ignored2 = j2.dump(-1, ' ', false, json::error_handler_t::ignore); + s_ignored_ascii = j.dump(-1, ' ', true, json::error_handler_t::ignore); + s_ignored2_ascii = j2.dump(-1, ' ', true, json::error_handler_t::ignore); + s_replaced = j.dump(-1, ' ', false, json::error_handler_t::replace); + s_replaced2 = j2.dump(-1, ' ', false, json::error_handler_t::replace); + s_replaced_ascii = j.dump(-1, ' ', true, json::error_handler_t::replace); + s_replaced2_ascii = j2.dump(-1, ' ', true, json::error_handler_t::replace); + + if (success_expected) + { + static std::string s_strict; + // strict mode must not throw if success is expected + s_strict = j.dump(); + // all dumps should agree on the string + CHECK(s_strict == s_ignored); + CHECK(s_strict == s_replaced); + } + else + { + // strict mode must throw if success is not expected + // dump() is nodiscard; the exception is thrown by dump() itself before it would return + CHECK_THROWS_AS(utils::ignore_return_value(j.dump()), json::type_error&); + // ignore and replace must create different dumps + CHECK(s_ignored != s_replaced); + + // check that replace string contains a replacement character + CHECK(s_replaced.find("\xEF\xBF\xBD") != std::string::npos); + } + + // check that prefix and suffix are preserved + CHECK(s_ignored2.substr(1, 3) == "abc"); + CHECK(s_ignored2.substr(s_ignored2.size() - 4, 3) == "xyz"); + CHECK(s_ignored2_ascii.substr(1, 3) == "abc"); + CHECK(s_ignored2_ascii.substr(s_ignored2_ascii.size() - 4, 3) == "xyz"); + CHECK(s_replaced2.substr(1, 3) == "abc"); + CHECK(s_replaced2.substr(s_replaced2.size() - 4, 3) == "xyz"); + CHECK(s_replaced2_ascii.substr(1, 3) == "abc"); + CHECK(s_replaced2_ascii.substr(s_replaced2_ascii.size() - 4, 3) == "xyz"); +} + +void check_utf8string(bool success_expected, int byte1, int byte2, int byte3, int byte4); + +// create and check a JSON string with up to four UTF-8 bytes +void check_utf8string(bool success_expected, int byte1, int byte2 = -1, int byte3 = -1, int byte4 = -1) +{ + static std::string json_string; + json_string = "\""; + + CAPTURE(byte1) + json_string += std::string(1, static_cast(byte1)); + + if (byte2 != -1) + { + CAPTURE(byte2) + json_string += std::string(1, static_cast(byte2)); + } + + if (byte3 != -1) + { + CAPTURE(byte3) + json_string += std::string(1, static_cast(byte3)); + } + + if (byte4 != -1) + { + CAPTURE(byte4) + json_string += std::string(1, static_cast(byte4)); + } + + json_string += "\""; + + CAPTURE(json_string) + + json _; + if (success_expected) + { + CHECK_NOTHROW(_ = json::parse(json_string)); + } + else + { + CHECK_THROWS_AS(_ = json::parse(json_string), json::parse_error&); + } +} +} // namespace + +TEST_CASE("Unicode") +{ + SECTION("\\uxxxx sequences") + { + // create an escaped string from a code point + const auto codepoint_to_unicode = [](std::size_t cp) + { + // code points are represented as a six-character sequence: a + // reverse solidus, followed by the lowercase letter u, followed + // by four hexadecimal digits that encode the character's code + // point; formatted by hand as this is called ~1.1M times (#5418) + std::string result = "\\u"; + for (int shift = 12; shift >= 0; shift -= 4) + { + result += "0123456789abcdef"[(cp >> shift) & 0xFu]; + } + return result; + }; + + SECTION("correct sequences") + { + // generate all UTF-8 code points; in total, 1112064 code points are + // generated: 0x1FFFFF code points - 2048 invalid values between + // 0xD800 and 0xDFFF. + for (std::size_t cp = 0; cp <= 0x10FFFFu; ++cp) + { + // string to store the code point as in \uxxxx format + std::string json_text = "\""; + + // decide whether to use one or two \uxxxx sequences + if (cp < 0x10000u) + { + // The Unicode standard permanently reserves these code point + // values for UTF-16 encoding of the high and low surrogates, and + // they will never be assigned a character, so there should be no + // reason to encode them. The official Unicode standard says that + // no UTF forms, including UTF-16, can encode these code points. + if (cp >= 0xD800u && cp <= 0xDFFFu) + { + // if we would not skip these code points, we would get a + // "missing low surrogate" exception + continue; + } + + // code points in the Basic Multilingual Plane can be + // represented with one \uxxxx sequence + json_text += codepoint_to_unicode(cp); + } + else + { + // To escape an extended character that is not in the Basic + // Multilingual Plane, the character is represented as a + // 12-character sequence, encoding the UTF-16 surrogate pair + const auto codepoint1 = 0xd800u + (((cp - 0x10000u) >> 10) & 0x3ffu); + const auto codepoint2 = 0xdc00u + ((cp - 0x10000u) & 0x3ffu); + json_text += codepoint_to_unicode(codepoint1) + codepoint_to_unicode(codepoint2); + } + + json_text += "\""; + CAPTURE(json_text) + json _; + CHECK_NOTHROW(_ = json::parse(json_text)); + } + } + + SECTION("incorrect sequences") + { + SECTION("incorrect surrogate values") + { + json _; + + CHECK_THROWS_WITH_AS(_ = json::parse("\"\\uDC00\\uDC00\""), "[json.exception.parse_error.101] parse error at line 1, column 7: syntax error while parsing value - invalid string: surrogate U+DC00..U+DFFF must follow U+D800..U+DBFF; last read: '\"\\uDC00'", json::parse_error&); + + CHECK_THROWS_WITH_AS(_ = json::parse("\"\\uD7FF\\uDC00\""), "[json.exception.parse_error.101] parse error at line 1, column 13: syntax error while parsing value - invalid string: surrogate U+DC00..U+DFFF must follow U+D800..U+DBFF; last read: '\"\\uD7FF\\uDC00'", json::parse_error&); + + CHECK_THROWS_WITH_AS(_ = json::parse("\"\\uD800]\""), "[json.exception.parse_error.101] parse error at line 1, column 8: syntax error while parsing value - invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF; last read: '\"\\uD800]'", json::parse_error&); + + CHECK_THROWS_WITH_AS(_ = json::parse("\"\\uD800\\v\""), "[json.exception.parse_error.101] parse error at line 1, column 9: syntax error while parsing value - invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF; last read: '\"\\uD800\\v'", json::parse_error&); + + CHECK_THROWS_WITH_AS(_ = json::parse("\"\\uD800\\u123\""), "[json.exception.parse_error.101] parse error at line 1, column 13: syntax error while parsing value - invalid string: '\\u' must be followed by 4 hex digits; last read: '\"\\uD800\\u123\"'", json::parse_error&); + + CHECK_THROWS_WITH_AS(_ = json::parse("\"\\uD800\\uDBFF\""), "[json.exception.parse_error.101] parse error at line 1, column 13: syntax error while parsing value - invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF; last read: '\"\\uD800\\uDBFF'", json::parse_error&); + + CHECK_THROWS_WITH_AS(_ = json::parse("\"\\uD800\\uE000\""), "[json.exception.parse_error.101] parse error at line 1, column 13: syntax error while parsing value - invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF; last read: '\"\\uD800\\uE000'", json::parse_error&); + } + } + +#if 0 // NOLINT(readability-avoid-unconditional-preprocessor-if) + SECTION("incorrect sequences") + { + SECTION("high surrogate without low surrogate") + { + // D800..DBFF are high surrogates and must be followed by low + // surrogates DC00..DFFF; here, nothing follows + for (std::size_t cp = 0xD800u; cp <= 0xDBFFu; ++cp) + { + std::string json_text = "\"" + codepoint_to_unicode(cp) + "\""; + CAPTURE(json_text) + CHECK_THROWS_AS(json::parse(json_text), json::parse_error&); + } + } + + SECTION("high surrogate with wrong low surrogate") + { + // D800..DBFF are high surrogates and must be followed by low + // surrogates DC00..DFFF; here a different sequence follows + for (std::size_t cp1 = 0xD800u; cp1 <= 0xDBFFu; ++cp1) + { + for (std::size_t cp2 = 0x0000u; cp2 <= 0xFFFFu; ++cp2) + { + if (0xDC00u <= cp2 && cp2 <= 0xDFFFu) + { + continue; + } + + std::string json_text = "\"" + codepoint_to_unicode(cp1) + codepoint_to_unicode(cp2) + "\""; + CAPTURE(json_text) + CHECK_THROWS_AS(json::parse(json_text), json::parse_error&); + } + } + } + + SECTION("low surrogate without high surrogate") + { + // low surrogates DC00..DFFF must follow high surrogates; here, + // they occur alone + for (std::size_t cp = 0xDC00u; cp <= 0xDFFFu; ++cp) + { + std::string json_text = "\"" + codepoint_to_unicode(cp) + "\""; + CAPTURE(json_text) + CHECK_THROWS_AS(json::parse(json_text), json::parse_error&); + } + } + + } +#endif + } + + SECTION("read all unicode characters") + { + // read a file with all Unicode characters stored as single-character + // strings in a JSON array + std::ifstream f(TEST_DATA_DIRECTORY "/json_nlohmann_tests/all_unicode.json"); + json j; + CHECK_NOTHROW(f >> j); + + // the array has 1112064 + 1 elements (a terminating "null" value) + // Note: 1112064 = 0x1FFFFF code points - 2048 invalid values between + // 0xD800 and 0xDFFF. + CHECK(j.size() == 1112065); + + SECTION("check JSON Pointers") + { + for (const auto& s : j) + { + // skip non-string JSON values + if (!s.is_string()) + { + continue; + } + + auto ptr = s.get(); + + // tilde must be followed by 0 or 1 + if (ptr == "~") + { + ptr += "0"; + } + + // JSON Pointers must begin with "/" + ptr.insert(0, "/"); + + CHECK_NOTHROW(json::json_pointer("/" + ptr)); + + // check escape/unescape roundtrip + auto escaped = nlohmann::detail::escape(ptr); + nlohmann::detail::unescape(escaped); + CHECK(escaped == ptr); + } + } + } + + SECTION("ignore byte-order-mark") + { + SECTION("in a stream") + { + // read a file with a UTF-8 BOM + std::ifstream f(TEST_DATA_DIRECTORY "/json_nlohmann_tests/bom.json"); + json j; + CHECK_NOTHROW(f >> j); + } + + SECTION("with an iterator") + { + std::string i = "\xef\xbb\xbf{\n \"foo\": true\n}"; + json _; + CHECK_NOTHROW(_ = json::parse(i.begin(), i.end())); + } + } + + SECTION("error for incomplete/wrong BOM") + { + json _; + CHECK_THROWS_AS(_ = json::parse("\xef\xbb"), json::parse_error&); + CHECK_THROWS_AS(_ = json::parse("\xef\xbb\xbb"), json::parse_error&); + } +} + +namespace +{ +void roundtrip(bool success_expected, const std::string& s); + +void roundtrip(bool success_expected, const std::string& s) +{ + CAPTURE(s) + json _; + + // create JSON string value + const json j = s; + // create JSON text + const std::string ps = std::string("\"") + s + "\""; + + if (success_expected) + { + // serialization succeeds + // dump() is nodiscard; this only checks that dumping does not throw + CHECK_NOTHROW(utils::ignore_return_value(j.dump())); + + // exclude parse test for U+0000 + if (s[0] != '\0') + { + // parsing JSON text succeeds + CHECK_NOTHROW(_ = json::parse(ps)); + } + + // roundtrip succeeds + CHECK_NOTHROW(_ = json::parse(j.dump())); + + // after roundtrip, the same string is stored + const json jr = json::parse(j.dump()); + CHECK(jr.get() == s); + } + else + { + // serialization fails + // dump() is nodiscard; the exception is thrown by dump() itself before it would return + CHECK_THROWS_AS(utils::ignore_return_value(j.dump()), json::type_error&); + + // parsing JSON text fails + CHECK_THROWS_AS(_ = json::parse(ps), json::parse_error&); + } +} +} // namespace + +TEST_CASE("Markus Kuhn's UTF-8 decoder capability and stress test") +{ + // Markus Kuhn - 2015-08-28 - CC BY 4.0 + // http://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt + + SECTION("1 Some correct UTF-8 text") + { + roundtrip(true, "κόσμε"); + } + + SECTION("2 Boundary condition test cases") + { + SECTION("2.1 First possible sequence of a certain length") + { + // 2.1.1 1 byte (U-00000000) + roundtrip(true, std::string("\0", 1)); + // 2.1.2 2 bytes (U-00000080) + roundtrip(true, "\xc2\x80"); + // 2.1.3 3 bytes (U-00000800) + roundtrip(true, "\xe0\xa0\x80"); + // 2.1.4 4 bytes (U-00010000) + roundtrip(true, "\xf0\x90\x80\x80"); + + // 2.1.5 5 bytes (U-00200000) + roundtrip(false, "\xF8\x88\x80\x80\x80"); + // 2.1.6 6 bytes (U-04000000) + roundtrip(false, "\xFC\x84\x80\x80\x80\x80"); + } + + SECTION("2.2 Last possible sequence of a certain length") + { + // 2.2.1 1 byte (U-0000007F) + roundtrip(true, "\x7f"); + // 2.2.2 2 bytes (U-000007FF) + roundtrip(true, "\xdf\xbf"); + // 2.2.3 3 bytes (U-0000FFFF) + roundtrip(true, "\xef\xbf\xbf"); + + // 2.2.4 4 bytes (U-001FFFFF) + roundtrip(false, "\xF7\xBF\xBF\xBF"); + // 2.2.5 5 bytes (U-03FFFFFF) + roundtrip(false, "\xFB\xBF\xBF\xBF\xBF"); + // 2.2.6 6 bytes (U-7FFFFFFF) + roundtrip(false, "\xFD\xBF\xBF\xBF\xBF\xBF"); + } + + SECTION("2.3 Other boundary conditions") + { + // 2.3.1 U-0000D7FF = ed 9f bf + roundtrip(true, "\xed\x9f\xbf"); + // 2.3.2 U-0000E000 = ee 80 80 + roundtrip(true, "\xee\x80\x80"); + // 2.3.3 U-0000FFFD = ef bf bd + roundtrip(true, "\xef\xbf\xbd"); + // 2.3.4 U-0010FFFF = f4 8f bf bf + roundtrip(true, "\xf4\x8f\xbf\xbf"); + + // 2.3.5 U-00110000 = f4 90 80 80 + roundtrip(false, "\xf4\x90\x80\x80"); + } + } + + SECTION("3 Malformed sequences") + { + SECTION("3.1 Unexpected continuation bytes") + { + // Each unexpected continuation byte should be separately signalled as a + // malformed sequence of its own. + + // 3.1.1 First continuation byte 0x80 + roundtrip(false, "\x80"); + // 3.1.2 Last continuation byte 0xbf + roundtrip(false, "\xbf"); + + // 3.1.3 2 continuation bytes + roundtrip(false, "\x80\xbf"); + // 3.1.4 3 continuation bytes + roundtrip(false, "\x80\xbf\x80"); + // 3.1.5 4 continuation bytes + roundtrip(false, "\x80\xbf\x80\xbf"); + // 3.1.6 5 continuation bytes + roundtrip(false, "\x80\xbf\x80\xbf\x80"); + // 3.1.7 6 continuation bytes + roundtrip(false, "\x80\xbf\x80\xbf\x80\xbf"); + // 3.1.8 7 continuation bytes + roundtrip(false, "\x80\xbf\x80\xbf\x80\xbf\x80"); + + // 3.1.9 Sequence of all 64 possible continuation bytes (0x80-0xbf) + roundtrip(false, "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"); + } + + SECTION("3.2 Lonely start characters") + { + // 3.2.1 All 32 first bytes of 2-byte sequences (0xc0-0xdf) + roundtrip(false, "\xc0 \xc1 \xc2 \xc3 \xc4 \xc5 \xc6 \xc7 \xc8 \xc9 \xca \xcb \xcc \xcd \xce \xcf \xd0 \xd1 \xd2 \xd3 \xd4 \xd5 \xd6 \xd7 \xd8 \xd9 \xda \xdb \xdc \xdd \xde \xdf"); + // 3.2.2 All 16 first bytes of 3-byte sequences (0xe0-0xef) + roundtrip(false, "\xe0 \xe1 \xe2 \xe3 \xe4 \xe5 \xe6 \xe7 \xe8 \xe9 \xea \xeb \xec \xed \xee \xef"); + // 3.2.3 All 8 first bytes of 4-byte sequences (0xf0-0xf7) + roundtrip(false, "\xf0 \xf1 \xf2 \xf3 \xf4 \xf5 \xf6 \xf7"); + // 3.2.4 All 4 first bytes of 5-byte sequences (0xf8-0xfb) + roundtrip(false, "\xf8 \xf9 \xfa \xfb"); + // 3.2.5 All 2 first bytes of 6-byte sequences (0xfc-0xfd) + roundtrip(false, "\xfc \xfd"); + } + + SECTION("3.3 Sequences with last continuation byte missing") + { + // All bytes of an incomplete sequence should be signalled as a single + // malformed sequence, i.e., you should see only a single replacement + // character in each of the next 10 tests. (Characters as in section 2) + + // 3.3.1 2-byte sequence with last byte missing (U+0000) + roundtrip(false, "\xc0"); + // 3.3.2 3-byte sequence with last byte missing (U+0000) + roundtrip(false, "\xe0\x80"); + // 3.3.3 4-byte sequence with last byte missing (U+0000) + roundtrip(false, "\xf0\x80\x80"); + // 3.3.4 5-byte sequence with last byte missing (U+0000) + roundtrip(false, "\xf8\x80\x80\x80"); + // 3.3.5 6-byte sequence with last byte missing (U+0000) + roundtrip(false, "\xfc\x80\x80\x80\x80"); + // 3.3.6 2-byte sequence with last byte missing (U-000007FF) + roundtrip(false, "\xdf"); + // 3.3.7 3-byte sequence with last byte missing (U-0000FFFF) + roundtrip(false, "\xef\xbf"); + // 3.3.8 4-byte sequence with last byte missing (U-001FFFFF) + roundtrip(false, "\xf7\xbf\xbf"); + // 3.3.9 5-byte sequence with last byte missing (U-03FFFFFF) + roundtrip(false, "\xfb\xbf\xbf\xbf"); + // 3.3.10 6-byte sequence with last byte missing (U-7FFFFFFF) + roundtrip(false, "\xfd\xbf\xbf\xbf\xbf"); + } + + SECTION("3.4 Concatenation of incomplete sequences") + { + // All the 10 sequences of 3.3 concatenated, you should see 10 malformed + // sequences being signalled: + roundtrip(false, "\xc0\xe0\x80\xf0\x80\x80\xf8\x80\x80\x80\xfc\x80\x80\x80\x80\xdf\xef\xbf\xf7\xbf\xbf\xfb\xbf\xbf\xbf\xfd\xbf\xbf\xbf\xbf"); + } + + SECTION("3.5 Impossible bytes") + { + // The following two bytes cannot appear in a correct UTF-8 string + + // 3.5.1 fe + roundtrip(false, "\xfe"); + // 3.5.2 ff + roundtrip(false, "\xff"); + // 3.5.3 fe fe ff ff + roundtrip(false, "\xfe\xfe\xff\xff"); + } + } + + SECTION("4 Overlong sequences") + { + // The following sequences are not malformed according to the letter of + // the Unicode 2.0 standard. However, they are longer then necessary and + // a correct UTF-8 encoder is not allowed to produce them. A "safe UTF-8 + // decoder" should reject them just like malformed sequences for two + // reasons: (1) It helps to debug applications if overlong sequences are + // not treated as valid representations of characters, because this helps + // to spot problems more quickly. (2) Overlong sequences provide + // alternative representations of characters, that could maliciously be + // used to bypass filters that check only for ASCII characters. For + // instance, a 2-byte encoded line feed (LF) would not be caught by a + // line counter that counts only 0x0a bytes, but it would still be + // processed as a line feed by an unsafe UTF-8 decoder later in the + // pipeline. From a security point of view, ASCII compatibility of UTF-8 + // sequences means also, that ASCII characters are *only* allowed to be + // represented by ASCII bytes in the range 0x00-0x7f. To ensure this + // aspect of ASCII compatibility, use only "safe UTF-8 decoders" that + // reject overlong UTF-8 sequences for which a shorter encoding exists. + + SECTION("4.1 Examples of an overlong ASCII character") + { + // With a safe UTF-8 decoder, all the following five overlong + // representations of the ASCII character slash ("/") should be rejected + // like a malformed UTF-8 sequence, for instance by substituting it with + // a replacement character. If you see a slash below, you do not have a + // safe UTF-8 decoder! + + // 4.1.1 U+002F = c0 af + roundtrip(false, "\xc0\xaf"); + // 4.1.2 U+002F = e0 80 af + roundtrip(false, "\xe0\x80\xaf"); + // 4.1.3 U+002F = f0 80 80 af + roundtrip(false, "\xf0\x80\x80\xaf"); + // 4.1.4 U+002F = f8 80 80 80 af + roundtrip(false, "\xf8\x80\x80\x80\xaf"); + // 4.1.5 U+002F = fc 80 80 80 80 af + roundtrip(false, "\xfc\x80\x80\x80\x80\xaf"); + } + + SECTION("4.2 Maximum overlong sequences") + { + // Below you see the highest Unicode value that is still resulting in an + // overlong sequence if represented with the given number of bytes. This + // is a boundary test for safe UTF-8 decoders. All five characters should + // be rejected like malformed UTF-8 sequences. + + // 4.2.1 U-0000007F = c1 bf + roundtrip(false, "\xc1\xbf"); + // 4.2.2 U-000007FF = e0 9f bf + roundtrip(false, "\xe0\x9f\xbf"); + // 4.2.3 U-0000FFFF = f0 8f bf bf + roundtrip(false, "\xf0\x8f\xbf\xbf"); + // 4.2.4 U-001FFFFF = f8 87 bf bf bf + roundtrip(false, "\xf8\x87\xbf\xbf\xbf"); + // 4.2.5 U-03FFFFFF = fc 83 bf bf bf bf + roundtrip(false, "\xfc\x83\xbf\xbf\xbf\xbf"); + } + + SECTION("4.3 Overlong representation of the NUL character") + { + // The following five sequences should also be rejected like malformed + // UTF-8 sequences and should not be treated like the ASCII NUL + // character. + + // 4.3.1 U+0000 = c0 80 + roundtrip(false, "\xc0\x80"); + // 4.3.2 U+0000 = e0 80 80 + roundtrip(false, "\xe0\x80\x80"); + // 4.3.3 U+0000 = f0 80 80 80 + roundtrip(false, "\xf0\x80\x80\x80"); + // 4.3.4 U+0000 = f8 80 80 80 80 + roundtrip(false, "\xf8\x80\x80\x80\x80"); + // 4.3.5 U+0000 = fc 80 80 80 80 80 + roundtrip(false, "\xfc\x80\x80\x80\x80\x80"); + } + } + + SECTION("5 Illegal code positions") + { + // The following UTF-8 sequences should be rejected like malformed + // sequences, because they never represent valid ISO 10646 characters and + // a UTF-8 decoder that accepts them might introduce security problems + // comparable to overlong UTF-8 sequences. + + SECTION("5.1 Single UTF-16 surrogates") + { + // 5.1.1 U+D800 = ed a0 80 + roundtrip(false, "\xed\xa0\x80"); + // 5.1.2 U+DB7F = ed ad bf + roundtrip(false, "\xed\xad\xbf"); + // 5.1.3 U+DB80 = ed ae 80 + roundtrip(false, "\xed\xae\x80"); + // 5.1.4 U+DBFF = ed af bf + roundtrip(false, "\xed\xaf\xbf"); + // 5.1.5 U+DC00 = ed b0 80 + roundtrip(false, "\xed\xb0\x80"); + // 5.1.6 U+DF80 = ed be 80 + roundtrip(false, "\xed\xbe\x80"); + // 5.1.7 U+DFFF = ed bf bf + roundtrip(false, "\xed\xbf\xbf"); + } + + SECTION("5.2 Paired UTF-16 surrogates") + { + // 5.2.1 U+D800 U+DC00 = ed a0 80 ed b0 80 + roundtrip(false, "\xed\xa0\x80\xed\xb0\x80"); + // 5.2.2 U+D800 U+DFFF = ed a0 80 ed bf bf + roundtrip(false, "\xed\xa0\x80\xed\xbf\xbf"); + // 5.2.3 U+DB7F U+DC00 = ed ad bf ed b0 80 + roundtrip(false, "\xed\xad\xbf\xed\xb0\x80"); + // 5.2.4 U+DB7F U+DFFF = ed ad bf ed bf bf + roundtrip(false, "\xed\xad\xbf\xed\xbf\xbf"); + // 5.2.5 U+DB80 U+DC00 = ed ae 80 ed b0 80 + roundtrip(false, "\xed\xae\x80\xed\xb0\x80"); + // 5.2.6 U+DB80 U+DFFF = ed ae 80 ed bf bf + roundtrip(false, "\xed\xae\x80\xed\xbf\xbf"); + // 5.2.7 U+DBFF U+DC00 = ed af bf ed b0 80 + roundtrip(false, "\xed\xaf\xbf\xed\xb0\x80"); + // 5.2.8 U+DBFF U+DFFF = ed af bf ed bf bf + roundtrip(false, "\xed\xaf\xbf\xed\xbf\xbf"); + } + + SECTION("5.3 Noncharacter code positions") + { + // The following "noncharacters" are "reserved for internal use" by + // applications, and according to older versions of the Unicode Standard + // "should never be interchanged". Unicode Corrigendum #9 dropped the + // latter restriction. Nevertheless, their presence in incoming UTF-8 data + // can remain a potential security risk, depending on what use is made of + // these codes subsequently. Examples of such internal use: + // + // - Some file APIs with 16-bit characters may use the integer value -1 + // = U+FFFF to signal an end-of-file (EOF) or error condition. + // + // - In some UTF-16 receivers, code point U+FFFE might trigger a + // byte-swap operation (to convert between UTF-16LE and UTF-16BE). + // + // With such internal use of noncharacters, it may be desirable and safer + // to block those code points in UTF-8 decoders, as they should never + // occur legitimately in incoming UTF-8 data, and could trigger unsafe + // behaviour in subsequent processing. + + // Particularly problematic noncharacters in 16-bit applications: + + // 5.3.1 U+FFFE = ef bf be + roundtrip(true, "\xef\xbf\xbe"); + // 5.3.2 U+FFFF = ef bf bf + roundtrip(true, "\xef\xbf\xbf"); + + // 5.3.3 U+FDD0 .. U+FDEF + roundtrip(true, "\xEF\xB7\x90"); + roundtrip(true, "\xEF\xB7\x91"); + roundtrip(true, "\xEF\xB7\x92"); + roundtrip(true, "\xEF\xB7\x93"); + roundtrip(true, "\xEF\xB7\x94"); + roundtrip(true, "\xEF\xB7\x95"); + roundtrip(true, "\xEF\xB7\x96"); + roundtrip(true, "\xEF\xB7\x97"); + roundtrip(true, "\xEF\xB7\x98"); + roundtrip(true, "\xEF\xB7\x99"); + roundtrip(true, "\xEF\xB7\x9A"); + roundtrip(true, "\xEF\xB7\x9B"); + roundtrip(true, "\xEF\xB7\x9C"); + roundtrip(true, "\xEF\xB7\x9D"); + roundtrip(true, "\xEF\xB7\x9E"); + roundtrip(true, "\xEF\xB7\x9F"); + roundtrip(true, "\xEF\xB7\xA0"); + roundtrip(true, "\xEF\xB7\xA1"); + roundtrip(true, "\xEF\xB7\xA2"); + roundtrip(true, "\xEF\xB7\xA3"); + roundtrip(true, "\xEF\xB7\xA4"); + roundtrip(true, "\xEF\xB7\xA5"); + roundtrip(true, "\xEF\xB7\xA6"); + roundtrip(true, "\xEF\xB7\xA7"); + roundtrip(true, "\xEF\xB7\xA8"); + roundtrip(true, "\xEF\xB7\xA9"); + roundtrip(true, "\xEF\xB7\xAA"); + roundtrip(true, "\xEF\xB7\xAB"); + roundtrip(true, "\xEF\xB7\xAC"); + roundtrip(true, "\xEF\xB7\xAD"); + roundtrip(true, "\xEF\xB7\xAE"); + roundtrip(true, "\xEF\xB7\xAF"); + + // 5.3.4 U+nFFFE U+nFFFF (for n = 1..10) + roundtrip(true, "\xF0\x9F\xBF\xBF"); + roundtrip(true, "\xF0\xAF\xBF\xBF"); + roundtrip(true, "\xF0\xBF\xBF\xBF"); + roundtrip(true, "\xF1\x8F\xBF\xBF"); + roundtrip(true, "\xF1\x9F\xBF\xBF"); + roundtrip(true, "\xF1\xAF\xBF\xBF"); + roundtrip(true, "\xF1\xBF\xBF\xBF"); + roundtrip(true, "\xF2\x8F\xBF\xBF"); + roundtrip(true, "\xF2\x9F\xBF\xBF"); + roundtrip(true, "\xF2\xAF\xBF\xBF"); + } + } +} + +TEST_CASE("Unicode RFC 3629") +{ + SECTION("RFC 3629") + { + /* + RFC 3629 describes in Sect. 4 the syntax of UTF-8 byte sequences as + follows: + + A UTF-8 string is a sequence of octets representing a sequence of UCS + characters. An octet sequence is valid UTF-8 only if it matches the + following syntax, which is derived from the rules for encoding UTF-8 + and is expressed in the ABNF of [RFC2234]. + + UTF8-octets = *( UTF8-char ) + UTF8-char = UTF8-1 / UTF8-2 / UTF8-3 / UTF8-4 + UTF8-1 = %x00-7F + UTF8-2 = %xC2-DF UTF8-tail + UTF8-3 = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2( UTF8-tail ) / + %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail ) + UTF8-4 = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) / + %xF4 %x80-8F 2( UTF8-tail ) + UTF8-tail = %x80-BF + */ + + SECTION("ill-formed first byte") + { + for (int byte1 = 0x80; byte1 <= 0xC1; ++byte1) + { + check_utf8string(false, byte1); + check_utf8dump(false, byte1); + } + + for (int byte1 = 0xF5; byte1 <= 0xFF; ++byte1) + { + check_utf8string(false, byte1); + check_utf8dump(false, byte1); + } + } + + SECTION("UTF8-1 (x00-x7F)") + { + SECTION("well-formed") + { + for (int byte1 = 0x00; byte1 <= 0x7F; ++byte1) + { + // unescaped control characters are parse errors in JSON + if (0x00 <= byte1 && byte1 <= 0x1F) + { + check_utf8string(false, byte1); + continue; + } + + // a single quote is a parse error in JSON + if (byte1 == 0x22) + { + check_utf8string(false, byte1); + continue; + } + + // a single backslash is a parse error in JSON + if (byte1 == 0x5C) + { + check_utf8string(false, byte1); + continue; + } + + // all other characters are OK + check_utf8string(true, byte1); + check_utf8dump(true, byte1); + } + } + } + + SECTION("UTF8-2 (xC2-xDF UTF8-tail)") + { + SECTION("well-formed") + { + for (int byte1 = 0xC2; byte1 <= 0xDF; ++byte1) + { + for (int byte2 = 0x80; byte2 <= 0xBF; ++byte2) + { + check_utf8string(true, byte1, byte2); + check_utf8dump(true, byte1, byte2); + } + } + } + + SECTION("ill-formed: missing second byte") + { + for (int byte1 = 0xC2; byte1 <= 0xDF; ++byte1) + { + check_utf8string(false, byte1); + check_utf8dump(false, byte1); + } + } + + SECTION("ill-formed: wrong second byte") + { + for (int byte1 = 0xC2; byte1 <= 0xDF; ++byte1) + { + for (int byte2 = 0x00; byte2 <= 0xFF; ++byte2) + { + // skip correct second byte + if (0x80 <= byte2 && byte2 <= 0xBF) + { + continue; + } + + check_utf8string(false, byte1, byte2); + check_utf8dump(false, byte1, byte2); + } + } + } + } + + SECTION("UTF8-3 (xE0 xA0-BF UTF8-tail)") + { + SECTION("well-formed") + { + for (int byte1 = 0xE0; byte1 <= 0xE0; ++byte1) + { + for (int byte2 = 0xA0; byte2 <= 0xBF; ++byte2) + { + for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3) + { + check_utf8string(true, byte1, byte2, byte3); + check_utf8dump(true, byte1, byte2, byte3); + } + } + } + } + + SECTION("ill-formed: missing second byte") + { + for (int byte1 = 0xE0; byte1 <= 0xE0; ++byte1) + { + check_utf8string(false, byte1); + check_utf8dump(false, byte1); + } + } + + SECTION("ill-formed: missing third byte") + { + for (int byte1 = 0xE0; byte1 <= 0xE0; ++byte1) + { + for (int byte2 = 0xA0; byte2 <= 0xBF; ++byte2) + { + check_utf8string(false, byte1, byte2); + check_utf8dump(false, byte1, byte2); + } + } + } + + SECTION("ill-formed: wrong second byte") + { + // the other bytes take the ends of each byte class, see utils::utf8_continuation_bytes (#5418) + for (int byte1 = 0xE0; byte1 <= 0xE0; ++byte1) + { + for (int byte2 = 0x00; byte2 <= 0xFF; ++byte2) + { + // skip correct second byte + if (0xA0 <= byte2 && byte2 <= 0xBF) + { + continue; + } + + for (const int byte3 : utils::utf8_continuation_bytes(0x80, 0xBF)) + { + check_utf8string(false, byte1, byte2, byte3); + check_utf8dump(false, byte1, byte2, byte3); + } + } + } + } + + SECTION("ill-formed: wrong third byte") + { + // the other bytes take the ends of each byte class, see utils::utf8_continuation_bytes (#5418) + for (int byte1 = 0xE0; byte1 <= 0xE0; ++byte1) + { + for (const int byte2 : utils::utf8_continuation_bytes(0xA0, 0xBF)) + { + for (int byte3 = 0x00; byte3 <= 0xFF; ++byte3) + { + // skip correct third byte + if (0x80 <= byte3 && byte3 <= 0xBF) + { + continue; + } + + check_utf8string(false, byte1, byte2, byte3); + check_utf8dump(false, byte1, byte2, byte3); + } + } + } + } + } + + SECTION("UTF8-3 (xE1-xEC UTF8-tail UTF8-tail)") + { + SECTION("well-formed") + { + for (int byte1 = 0xE1; byte1 <= 0xEC; ++byte1) + { + for (int byte2 = 0x80; byte2 <= 0xBF; ++byte2) + { + for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3) + { + check_utf8string(true, byte1, byte2, byte3); + check_utf8dump(true, byte1, byte2, byte3); + } + } + } + } + + SECTION("ill-formed: missing second byte") + { + for (int byte1 = 0xE1; byte1 <= 0xEC; ++byte1) + { + check_utf8string(false, byte1); + check_utf8dump(false, byte1); + } + } + + SECTION("ill-formed: missing third byte") + { + for (int byte1 = 0xE1; byte1 <= 0xEC; ++byte1) + { + for (int byte2 = 0x80; byte2 <= 0xBF; ++byte2) + { + check_utf8string(false, byte1, byte2); + check_utf8dump(false, byte1, byte2); + } + } + } + + SECTION("ill-formed: wrong second byte") + { + // the other bytes take the ends of each byte class, see utils::utf8_continuation_bytes (#5418) + for (int byte1 = 0xE1; byte1 <= 0xEC; ++byte1) + { + for (int byte2 = 0x00; byte2 <= 0xFF; ++byte2) + { + // skip correct second byte + if (0x80 <= byte2 && byte2 <= 0xBF) + { + continue; + } + + for (const int byte3 : utils::utf8_continuation_bytes(0x80, 0xBF)) + { + check_utf8string(false, byte1, byte2, byte3); + check_utf8dump(false, byte1, byte2, byte3); + } + } + } + } + + SECTION("ill-formed: wrong third byte") + { + // the other bytes take the ends of each byte class, see utils::utf8_continuation_bytes (#5418) + for (int byte1 = 0xE1; byte1 <= 0xEC; ++byte1) + { + for (const int byte2 : utils::utf8_continuation_bytes(0x80, 0xBF)) + { + for (int byte3 = 0x00; byte3 <= 0xFF; ++byte3) + { + // skip correct third byte + if (0x80 <= byte3 && byte3 <= 0xBF) + { + continue; + } + + check_utf8string(false, byte1, byte2, byte3); + check_utf8dump(false, byte1, byte2, byte3); + } + } + } + } + } + + SECTION("UTF8-3 (xED x80-9F UTF8-tail)") + { + SECTION("well-formed") + { + for (int byte1 = 0xED; byte1 <= 0xED; ++byte1) + { + for (int byte2 = 0x80; byte2 <= 0x9F; ++byte2) + { + for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3) + { + check_utf8string(true, byte1, byte2, byte3); + check_utf8dump(true, byte1, byte2, byte3); + } + } + } + } + + SECTION("ill-formed: missing second byte") + { + for (int byte1 = 0xED; byte1 <= 0xED; ++byte1) + { + check_utf8string(false, byte1); + check_utf8dump(false, byte1); + } + } + + SECTION("ill-formed: missing third byte") + { + for (int byte1 = 0xED; byte1 <= 0xED; ++byte1) + { + for (int byte2 = 0x80; byte2 <= 0x9F; ++byte2) + { + check_utf8string(false, byte1, byte2); + check_utf8dump(false, byte1, byte2); + } + } + } + + SECTION("ill-formed: wrong second byte") + { + // the other bytes take the ends of each byte class, see utils::utf8_continuation_bytes (#5418) + for (int byte1 = 0xED; byte1 <= 0xED; ++byte1) + { + for (int byte2 = 0x00; byte2 <= 0xFF; ++byte2) + { + // skip correct second byte + if (0x80 <= byte2 && byte2 <= 0x9F) + { + continue; + } + + for (const int byte3 : utils::utf8_continuation_bytes(0x80, 0xBF)) + { + check_utf8string(false, byte1, byte2, byte3); + check_utf8dump(false, byte1, byte2, byte3); + } + } + } + } + + SECTION("ill-formed: wrong third byte") + { + // the other bytes take the ends of each byte class, see utils::utf8_continuation_bytes (#5418) + for (int byte1 = 0xED; byte1 <= 0xED; ++byte1) + { + for (const int byte2 : utils::utf8_continuation_bytes(0x80, 0x9F)) + { + for (int byte3 = 0x00; byte3 <= 0xFF; ++byte3) + { + // skip correct third byte + if (0x80 <= byte3 && byte3 <= 0xBF) + { + continue; + } + + check_utf8string(false, byte1, byte2, byte3); + check_utf8dump(false, byte1, byte2, byte3); + } + } + } + } + } + + SECTION("UTF8-3 (xEE-xEF UTF8-tail UTF8-tail)") + { + SECTION("well-formed") + { + for (int byte1 = 0xEE; byte1 <= 0xEF; ++byte1) + { + for (int byte2 = 0x80; byte2 <= 0xBF; ++byte2) + { + for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3) + { + check_utf8string(true, byte1, byte2, byte3); + check_utf8dump(true, byte1, byte2, byte3); + } + } + } + } + + SECTION("ill-formed: missing second byte") + { + for (int byte1 = 0xEE; byte1 <= 0xEF; ++byte1) + { + check_utf8string(false, byte1); + check_utf8dump(false, byte1); + } + } + + SECTION("ill-formed: missing third byte") + { + for (int byte1 = 0xEE; byte1 <= 0xEF; ++byte1) + { + for (int byte2 = 0x80; byte2 <= 0xBF; ++byte2) + { + check_utf8string(false, byte1, byte2); + check_utf8dump(false, byte1, byte2); + } + } + } + + SECTION("ill-formed: wrong second byte") + { + // the other bytes take the ends of each byte class, see utils::utf8_continuation_bytes (#5418) + for (int byte1 = 0xEE; byte1 <= 0xEF; ++byte1) + { + for (int byte2 = 0x00; byte2 <= 0xFF; ++byte2) + { + // skip correct second byte + if (0x80 <= byte2 && byte2 <= 0xBF) + { + continue; + } + + for (const int byte3 : utils::utf8_continuation_bytes(0x80, 0xBF)) + { + check_utf8string(false, byte1, byte2, byte3); + check_utf8dump(false, byte1, byte2, byte3); + } + } + } + } + + SECTION("ill-formed: wrong third byte") + { + // the other bytes take the ends of each byte class, see utils::utf8_continuation_bytes (#5418) + for (int byte1 = 0xEE; byte1 <= 0xEF; ++byte1) + { + for (const int byte2 : utils::utf8_continuation_bytes(0x80, 0xBF)) + { + for (int byte3 = 0x00; byte3 <= 0xFF; ++byte3) + { + // skip correct third byte + if (0x80 <= byte3 && byte3 <= 0xBF) + { + continue; + } + + check_utf8string(false, byte1, byte2, byte3); + check_utf8dump(false, byte1, byte2, byte3); + } + } + } + } + } + + SECTION("UTF8-4 (xF0 x90-BF UTF8-tail UTF8-tail)") + { + SECTION("well-formed") + { + for (int byte1 = 0xF0; byte1 <= 0xF0; ++byte1) + { + for (int byte2 = 0x90; byte2 <= 0xBF; ++byte2) + { + for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3) + { + for (int byte4 = 0x80; byte4 <= 0xBF; ++byte4) + { + check_utf8string(true, byte1, byte2, byte3, byte4); + check_utf8dump(true, byte1, byte2, byte3, byte4); + } + } + } + } + } + + SECTION("ill-formed: missing second byte") + { + for (int byte1 = 0xF0; byte1 <= 0xF0; ++byte1) + { + check_utf8string(false, byte1); + check_utf8dump(false, byte1); + } + } + + SECTION("ill-formed: missing third byte") + { + for (int byte1 = 0xF0; byte1 <= 0xF0; ++byte1) + { + for (int byte2 = 0x90; byte2 <= 0xBF; ++byte2) + { + check_utf8string(false, byte1, byte2); + check_utf8dump(false, byte1, byte2); + } + } + } + + SECTION("ill-formed: missing fourth byte") + { + for (int byte1 = 0xF0; byte1 <= 0xF0; ++byte1) + { + for (int byte2 = 0x90; byte2 <= 0xBF; ++byte2) + { + for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3) + { + check_utf8string(false, byte1, byte2, byte3); + check_utf8dump(false, byte1, byte2, byte3); + } + } + } + } + + SECTION("ill-formed: wrong second byte") + { + // the other bytes take the ends of each byte class, see utils::utf8_continuation_bytes (#5418) + for (int byte1 = 0xF0; byte1 <= 0xF0; ++byte1) + { + for (int byte2 = 0x00; byte2 <= 0xFF; ++byte2) + { + // skip correct second byte + if (0x90 <= byte2 && byte2 <= 0xBF) + { + continue; + } + + for (const int byte3 : utils::utf8_continuation_bytes(0x80, 0xBF)) + { + for (const int byte4 : utils::utf8_continuation_bytes(0x80, 0xBF)) + { + check_utf8string(false, byte1, byte2, byte3, byte4); + check_utf8dump(false, byte1, byte2, byte3, byte4); + } + } + } + } + } + + SECTION("ill-formed: wrong third byte") + { + // the other bytes take the ends of each byte class, see utils::utf8_continuation_bytes (#5418) + for (int byte1 = 0xF0; byte1 <= 0xF0; ++byte1) + { + for (const int byte2 : utils::utf8_continuation_bytes(0x90, 0xBF)) + { + for (int byte3 = 0x00; byte3 <= 0xFF; ++byte3) + { + // skip correct third byte + if (0x80 <= byte3 && byte3 <= 0xBF) + { + continue; + } + + for (const int byte4 : utils::utf8_continuation_bytes(0x80, 0xBF)) + { + check_utf8string(false, byte1, byte2, byte3, byte4); + check_utf8dump(false, byte1, byte2, byte3, byte4); + } + } + } + } + } + + SECTION("ill-formed: wrong fourth byte") + { + // the other bytes take the ends of each byte class, see utils::utf8_continuation_bytes (#5418) + for (int byte1 = 0xF0; byte1 <= 0xF0; ++byte1) + { + for (const int byte2 : utils::utf8_continuation_bytes(0x90, 0xBF)) + { + for (const int byte3 : utils::utf8_continuation_bytes(0x80, 0xBF)) + { + for (int byte4 = 0x00; byte4 <= 0xFF; ++byte4) + { + // skip correct fourth byte + if (0x80 <= byte4 && byte4 <= 0xBF) + { + continue; + } + + check_utf8string(false, byte1, byte2, byte3, byte4); + check_utf8dump(false, byte1, byte2, byte3, byte4); + } + } + } + } + } + } + + SECTION("UTF8-4 (xF1-F3 UTF8-tail UTF8-tail UTF8-tail)") + { + SECTION("well-formed") + { + for (int byte1 = 0xF1; byte1 <= 0xF3; ++byte1) + { + for (int byte2 = 0x80; byte2 <= 0xBF; ++byte2) + { + for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3) + { + for (int byte4 = 0x80; byte4 <= 0xBF; ++byte4) + { + check_utf8string(true, byte1, byte2, byte3, byte4); + check_utf8dump(true, byte1, byte2, byte3, byte4); + } + } + } + } + } + + SECTION("ill-formed: missing second byte") + { + for (int byte1 = 0xF1; byte1 <= 0xF3; ++byte1) + { + check_utf8string(false, byte1); + check_utf8dump(false, byte1); + } + } + + SECTION("ill-formed: missing third byte") + { + for (int byte1 = 0xF1; byte1 <= 0xF3; ++byte1) + { + for (int byte2 = 0x80; byte2 <= 0xBF; ++byte2) + { + check_utf8string(false, byte1, byte2); + check_utf8dump(false, byte1, byte2); + } + } + } + + SECTION("ill-formed: missing fourth byte") + { + for (int byte1 = 0xF1; byte1 <= 0xF3; ++byte1) + { + for (int byte2 = 0x80; byte2 <= 0xBF; ++byte2) + { + for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3) + { + check_utf8string(false, byte1, byte2, byte3); + check_utf8dump(false, byte1, byte2, byte3); + } + } + } + } + + SECTION("ill-formed: wrong second byte") + { + // the other bytes take the ends of each byte class, see utils::utf8_continuation_bytes (#5418) + for (int byte1 = 0xF1; byte1 <= 0xF3; ++byte1) + { + for (int byte2 = 0x00; byte2 <= 0xFF; ++byte2) + { + // skip correct second byte + if (0x80 <= byte2 && byte2 <= 0xBF) + { + continue; + } + + for (const int byte3 : utils::utf8_continuation_bytes(0x80, 0xBF)) + { + for (const int byte4 : utils::utf8_continuation_bytes(0x80, 0xBF)) + { + check_utf8string(false, byte1, byte2, byte3, byte4); + check_utf8dump(false, byte1, byte2, byte3, byte4); + } + } + } + } + } + + SECTION("ill-formed: wrong third byte") + { + // the other bytes take the ends of each byte class, see utils::utf8_continuation_bytes (#5418) + for (int byte1 = 0xF1; byte1 <= 0xF3; ++byte1) + { + for (const int byte2 : utils::utf8_continuation_bytes(0x80, 0xBF)) + { + for (int byte3 = 0x00; byte3 <= 0xFF; ++byte3) + { + // skip correct third byte + if (0x80 <= byte3 && byte3 <= 0xBF) + { + continue; + } + + for (const int byte4 : utils::utf8_continuation_bytes(0x80, 0xBF)) + { + check_utf8string(false, byte1, byte2, byte3, byte4); + check_utf8dump(false, byte1, byte2, byte3, byte4); + } + } + } + } + } + + SECTION("ill-formed: wrong fourth byte") + { + // the other bytes take the ends of each byte class, see utils::utf8_continuation_bytes (#5418) + for (int byte1 = 0xF1; byte1 <= 0xF3; ++byte1) + { + for (const int byte2 : utils::utf8_continuation_bytes(0x80, 0xBF)) + { + for (const int byte3 : utils::utf8_continuation_bytes(0x80, 0xBF)) + { + for (int byte4 = 0x00; byte4 <= 0xFF; ++byte4) + { + // skip correct fourth byte + if (0x80 <= byte4 && byte4 <= 0xBF) + { + continue; + } + + check_utf8string(false, byte1, byte2, byte3, byte4); + check_utf8dump(false, byte1, byte2, byte3, byte4); + } + } + } + } + } + } + + SECTION("UTF8-4 (xF4 x80-8F UTF8-tail UTF8-tail)") + { + SECTION("well-formed") + { + for (int byte1 = 0xF4; byte1 <= 0xF4; ++byte1) + { + for (int byte2 = 0x80; byte2 <= 0x8F; ++byte2) + { + for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3) + { + for (int byte4 = 0x80; byte4 <= 0xBF; ++byte4) + { + check_utf8string(true, byte1, byte2, byte3, byte4); + check_utf8dump(true, byte1, byte2, byte3, byte4); + } + } + } + } + } + + SECTION("ill-formed: missing second byte") + { + for (int byte1 = 0xF4; byte1 <= 0xF4; ++byte1) + { + check_utf8string(false, byte1); + check_utf8dump(false, byte1); + } + } + + SECTION("ill-formed: missing third byte") + { + for (int byte1 = 0xF4; byte1 <= 0xF4; ++byte1) + { + for (int byte2 = 0x80; byte2 <= 0x8F; ++byte2) + { + check_utf8string(false, byte1, byte2); + check_utf8dump(false, byte1, byte2); + } + } + } + + SECTION("ill-formed: missing fourth byte") + { + for (int byte1 = 0xF4; byte1 <= 0xF4; ++byte1) + { + for (int byte2 = 0x80; byte2 <= 0x8F; ++byte2) + { + for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3) + { + check_utf8string(false, byte1, byte2, byte3); + check_utf8dump(false, byte1, byte2, byte3); + } + } + } + } + + SECTION("ill-formed: wrong second byte") + { + // the other bytes take the ends of each byte class, see utils::utf8_continuation_bytes (#5418) + for (int byte1 = 0xF4; byte1 <= 0xF4; ++byte1) + { + for (int byte2 = 0x00; byte2 <= 0xFF; ++byte2) + { + // skip correct second byte + if (0x80 <= byte2 && byte2 <= 0x8F) + { + continue; + } + + for (const int byte3 : utils::utf8_continuation_bytes(0x80, 0xBF)) + { + for (const int byte4 : utils::utf8_continuation_bytes(0x80, 0xBF)) + { + check_utf8string(false, byte1, byte2, byte3, byte4); + check_utf8dump(false, byte1, byte2, byte3, byte4); + } + } + } + } + } + + SECTION("ill-formed: wrong third byte") + { + // the other bytes take the ends of each byte class, see utils::utf8_continuation_bytes (#5418) + for (int byte1 = 0xF4; byte1 <= 0xF4; ++byte1) + { + for (const int byte2 : utils::utf8_continuation_bytes(0x80, 0x8F)) + { + for (int byte3 = 0x00; byte3 <= 0xFF; ++byte3) + { + // skip correct third byte + if (0x80 <= byte3 && byte3 <= 0xBF) + { + continue; + } + + for (const int byte4 : utils::utf8_continuation_bytes(0x80, 0xBF)) + { + check_utf8string(false, byte1, byte2, byte3, byte4); + check_utf8dump(false, byte1, byte2, byte3, byte4); + } + } + } + } + } + + SECTION("ill-formed: wrong fourth byte") + { + // the other bytes take the ends of each byte class, see utils::utf8_continuation_bytes (#5418) + for (int byte1 = 0xF4; byte1 <= 0xF4; ++byte1) + { + for (const int byte2 : utils::utf8_continuation_bytes(0x80, 0x8F)) + { + for (const int byte3 : utils::utf8_continuation_bytes(0x80, 0xBF)) + { + for (int byte4 = 0x00; byte4 <= 0xFF; ++byte4) + { + // skip correct fourth byte + if (0x80 <= byte4 && byte4 <= 0xBF) + { + continue; + } + + check_utf8string(false, byte1, byte2, byte3, byte4); + check_utf8dump(false, byte1, byte2, byte3, byte4); + } + } + } + } + } + } + } +} + +DOCTEST_CLANG_SUPPRESS_WARNING_POP diff --git a/tests/src/unit-unicode1.cpp b/tests/src/unit-unicode1.cpp deleted file mode 100644 index 2d744003a..000000000 --- a/tests/src/unit-unicode1.cpp +++ /dev/null @@ -1,623 +0,0 @@ -// __ _____ _____ _____ -// __| | __| | | | JSON for Modern C++ (supporting code) -// | | |__ | | | | | | version 3.12.0 -// |_____|_____|_____|_|___| https://github.com/nlohmann/json -// -// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann -// SPDX-License-Identifier: MIT - -#include "doctest_compatibility.h" - -// for some reason including this after the json header leads to linker errors with VS 2017... -#include -#include -using nlohmann::json; - -#include -#include -#include -#include "make_test_data_available.hpp" -#include "test_utils.hpp" - -TEST_CASE("Unicode (1/5)" * doctest::skip()) -{ - SECTION("\\uxxxx sequences") - { - // create an escaped string from a code point - const auto codepoint_to_unicode = [](std::size_t cp) - { - // code points are represented as a six-character sequence: a - // reverse solidus, followed by the lowercase letter u, followed - // by four hexadecimal digits that encode the character's code - // point - std::stringstream ss; - ss << "\\u" << std::setw(4) << std::setfill('0') << std::hex << cp; - return ss.str(); - }; - - SECTION("correct sequences") - { - // generate all UTF-8 code points; in total, 1112064 code points are - // generated: 0x1FFFFF code points - 2048 invalid values between - // 0xD800 and 0xDFFF. - for (std::size_t cp = 0; cp <= 0x10FFFFu; ++cp) - { - // string to store the code point as in \uxxxx format - std::string json_text = "\""; - - // decide whether to use one or two \uxxxx sequences - if (cp < 0x10000u) - { - // The Unicode standard permanently reserves these code point - // values for UTF-16 encoding of the high and low surrogates, and - // they will never be assigned a character, so there should be no - // reason to encode them. The official Unicode standard says that - // no UTF forms, including UTF-16, can encode these code points. - if (cp >= 0xD800u && cp <= 0xDFFFu) - { - // if we would not skip these code points, we would get a - // "missing low surrogate" exception - continue; - } - - // code points in the Basic Multilingual Plane can be - // represented with one \uxxxx sequence - json_text += codepoint_to_unicode(cp); - } - else - { - // To escape an extended character that is not in the Basic - // Multilingual Plane, the character is represented as a - // 12-character sequence, encoding the UTF-16 surrogate pair - const auto codepoint1 = 0xd800u + (((cp - 0x10000u) >> 10) & 0x3ffu); - const auto codepoint2 = 0xdc00u + ((cp - 0x10000u) & 0x3ffu); - json_text += codepoint_to_unicode(codepoint1) + codepoint_to_unicode(codepoint2); - } - - json_text += "\""; - CAPTURE(json_text) - json _; - CHECK_NOTHROW(_ = json::parse(json_text)); - } - } - - SECTION("incorrect sequences") - { - SECTION("incorrect surrogate values") - { - json _; - - CHECK_THROWS_WITH_AS(_ = json::parse("\"\\uDC00\\uDC00\""), "[json.exception.parse_error.101] parse error at line 1, column 7: syntax error while parsing value - invalid string: surrogate U+DC00..U+DFFF must follow U+D800..U+DBFF; last read: '\"\\uDC00'", json::parse_error&); - - CHECK_THROWS_WITH_AS(_ = json::parse("\"\\uD7FF\\uDC00\""), "[json.exception.parse_error.101] parse error at line 1, column 13: syntax error while parsing value - invalid string: surrogate U+DC00..U+DFFF must follow U+D800..U+DBFF; last read: '\"\\uD7FF\\uDC00'", json::parse_error&); - - CHECK_THROWS_WITH_AS(_ = json::parse("\"\\uD800]\""), "[json.exception.parse_error.101] parse error at line 1, column 8: syntax error while parsing value - invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF; last read: '\"\\uD800]'", json::parse_error&); - - CHECK_THROWS_WITH_AS(_ = json::parse("\"\\uD800\\v\""), "[json.exception.parse_error.101] parse error at line 1, column 9: syntax error while parsing value - invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF; last read: '\"\\uD800\\v'", json::parse_error&); - - CHECK_THROWS_WITH_AS(_ = json::parse("\"\\uD800\\u123\""), "[json.exception.parse_error.101] parse error at line 1, column 13: syntax error while parsing value - invalid string: '\\u' must be followed by 4 hex digits; last read: '\"\\uD800\\u123\"'", json::parse_error&); - - CHECK_THROWS_WITH_AS(_ = json::parse("\"\\uD800\\uDBFF\""), "[json.exception.parse_error.101] parse error at line 1, column 13: syntax error while parsing value - invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF; last read: '\"\\uD800\\uDBFF'", json::parse_error&); - - CHECK_THROWS_WITH_AS(_ = json::parse("\"\\uD800\\uE000\""), "[json.exception.parse_error.101] parse error at line 1, column 13: syntax error while parsing value - invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF; last read: '\"\\uD800\\uE000'", json::parse_error&); - } - } - -#if 0 // NOLINT(readability-avoid-unconditional-preprocessor-if) - SECTION("incorrect sequences") - { - SECTION("high surrogate without low surrogate") - { - // D800..DBFF are high surrogates and must be followed by low - // surrogates DC00..DFFF; here, nothing follows - for (std::size_t cp = 0xD800u; cp <= 0xDBFFu; ++cp) - { - std::string json_text = "\"" + codepoint_to_unicode(cp) + "\""; - CAPTURE(json_text) - CHECK_THROWS_AS(json::parse(json_text), json::parse_error&); - } - } - - SECTION("high surrogate with wrong low surrogate") - { - // D800..DBFF are high surrogates and must be followed by low - // surrogates DC00..DFFF; here a different sequence follows - for (std::size_t cp1 = 0xD800u; cp1 <= 0xDBFFu; ++cp1) - { - for (std::size_t cp2 = 0x0000u; cp2 <= 0xFFFFu; ++cp2) - { - if (0xDC00u <= cp2 && cp2 <= 0xDFFFu) - { - continue; - } - - std::string json_text = "\"" + codepoint_to_unicode(cp1) + codepoint_to_unicode(cp2) + "\""; - CAPTURE(json_text) - CHECK_THROWS_AS(json::parse(json_text), json::parse_error&); - } - } - } - - SECTION("low surrogate without high surrogate") - { - // low surrogates DC00..DFFF must follow high surrogates; here, - // they occur alone - for (std::size_t cp = 0xDC00u; cp <= 0xDFFFu; ++cp) - { - std::string json_text = "\"" + codepoint_to_unicode(cp) + "\""; - CAPTURE(json_text) - CHECK_THROWS_AS(json::parse(json_text), json::parse_error&); - } - } - - } -#endif - } - - SECTION("read all unicode characters") - { - // read a file with all Unicode characters stored as single-character - // strings in a JSON array - std::ifstream f(TEST_DATA_DIRECTORY "/json_nlohmann_tests/all_unicode.json"); - json j; - CHECK_NOTHROW(f >> j); - - // the array has 1112064 + 1 elements (a terminating "null" value) - // Note: 1112064 = 0x1FFFFF code points - 2048 invalid values between - // 0xD800 and 0xDFFF. - CHECK(j.size() == 1112065); - - SECTION("check JSON Pointers") - { - for (const auto& s : j) - { - // skip non-string JSON values - if (!s.is_string()) - { - continue; - } - - auto ptr = s.get(); - - // tilde must be followed by 0 or 1 - if (ptr == "~") - { - ptr += "0"; - } - - // JSON Pointers must begin with "/" - ptr.insert(0, "/"); - - CHECK_NOTHROW(json::json_pointer("/" + ptr)); - - // check escape/unescape roundtrip - auto escaped = nlohmann::detail::escape(ptr); - nlohmann::detail::unescape(escaped); - CHECK(escaped == ptr); - } - } - } - - SECTION("ignore byte-order-mark") - { - SECTION("in a stream") - { - // read a file with a UTF-8 BOM - std::ifstream f(TEST_DATA_DIRECTORY "/json_nlohmann_tests/bom.json"); - json j; - CHECK_NOTHROW(f >> j); - } - - SECTION("with an iterator") - { - std::string i = "\xef\xbb\xbf{\n \"foo\": true\n}"; - json _; - CHECK_NOTHROW(_ = json::parse(i.begin(), i.end())); - } - } - - SECTION("error for incomplete/wrong BOM") - { - json _; - CHECK_THROWS_AS(_ = json::parse("\xef\xbb"), json::parse_error&); - CHECK_THROWS_AS(_ = json::parse("\xef\xbb\xbb"), json::parse_error&); - } -} - -namespace -{ -void roundtrip(bool success_expected, const std::string& s); - -void roundtrip(bool success_expected, const std::string& s) -{ - CAPTURE(s) - json _; - - // create JSON string value - const json j = s; - // create JSON text - const std::string ps = std::string("\"") + s + "\""; - - if (success_expected) - { - // serialization succeeds - // dump() is nodiscard; this only checks that dumping does not throw - CHECK_NOTHROW(utils::ignore_return_value(j.dump())); - - // exclude parse test for U+0000 - if (s[0] != '\0') - { - // parsing JSON text succeeds - CHECK_NOTHROW(_ = json::parse(ps)); - } - - // roundtrip succeeds - CHECK_NOTHROW(_ = json::parse(j.dump())); - - // after roundtrip, the same string is stored - const json jr = json::parse(j.dump()); - CHECK(jr.get() == s); - } - else - { - // serialization fails - // dump() is nodiscard; the exception is thrown by dump() itself before it would return - CHECK_THROWS_AS(utils::ignore_return_value(j.dump()), json::type_error&); - - // parsing JSON text fails - CHECK_THROWS_AS(_ = json::parse(ps), json::parse_error&); - } -} -} // namespace - -TEST_CASE("Markus Kuhn's UTF-8 decoder capability and stress test") -{ - // Markus Kuhn - 2015-08-28 - CC BY 4.0 - // http://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt - - SECTION("1 Some correct UTF-8 text") - { - roundtrip(true, "κόσμε"); - } - - SECTION("2 Boundary condition test cases") - { - SECTION("2.1 First possible sequence of a certain length") - { - // 2.1.1 1 byte (U-00000000) - roundtrip(true, std::string("\0", 1)); - // 2.1.2 2 bytes (U-00000080) - roundtrip(true, "\xc2\x80"); - // 2.1.3 3 bytes (U-00000800) - roundtrip(true, "\xe0\xa0\x80"); - // 2.1.4 4 bytes (U-00010000) - roundtrip(true, "\xf0\x90\x80\x80"); - - // 2.1.5 5 bytes (U-00200000) - roundtrip(false, "\xF8\x88\x80\x80\x80"); - // 2.1.6 6 bytes (U-04000000) - roundtrip(false, "\xFC\x84\x80\x80\x80\x80"); - } - - SECTION("2.2 Last possible sequence of a certain length") - { - // 2.2.1 1 byte (U-0000007F) - roundtrip(true, "\x7f"); - // 2.2.2 2 bytes (U-000007FF) - roundtrip(true, "\xdf\xbf"); - // 2.2.3 3 bytes (U-0000FFFF) - roundtrip(true, "\xef\xbf\xbf"); - - // 2.2.4 4 bytes (U-001FFFFF) - roundtrip(false, "\xF7\xBF\xBF\xBF"); - // 2.2.5 5 bytes (U-03FFFFFF) - roundtrip(false, "\xFB\xBF\xBF\xBF\xBF"); - // 2.2.6 6 bytes (U-7FFFFFFF) - roundtrip(false, "\xFD\xBF\xBF\xBF\xBF\xBF"); - } - - SECTION("2.3 Other boundary conditions") - { - // 2.3.1 U-0000D7FF = ed 9f bf - roundtrip(true, "\xed\x9f\xbf"); - // 2.3.2 U-0000E000 = ee 80 80 - roundtrip(true, "\xee\x80\x80"); - // 2.3.3 U-0000FFFD = ef bf bd - roundtrip(true, "\xef\xbf\xbd"); - // 2.3.4 U-0010FFFF = f4 8f bf bf - roundtrip(true, "\xf4\x8f\xbf\xbf"); - - // 2.3.5 U-00110000 = f4 90 80 80 - roundtrip(false, "\xf4\x90\x80\x80"); - } - } - - SECTION("3 Malformed sequences") - { - SECTION("3.1 Unexpected continuation bytes") - { - // Each unexpected continuation byte should be separately signalled as a - // malformed sequence of its own. - - // 3.1.1 First continuation byte 0x80 - roundtrip(false, "\x80"); - // 3.1.2 Last continuation byte 0xbf - roundtrip(false, "\xbf"); - - // 3.1.3 2 continuation bytes - roundtrip(false, "\x80\xbf"); - // 3.1.4 3 continuation bytes - roundtrip(false, "\x80\xbf\x80"); - // 3.1.5 4 continuation bytes - roundtrip(false, "\x80\xbf\x80\xbf"); - // 3.1.6 5 continuation bytes - roundtrip(false, "\x80\xbf\x80\xbf\x80"); - // 3.1.7 6 continuation bytes - roundtrip(false, "\x80\xbf\x80\xbf\x80\xbf"); - // 3.1.8 7 continuation bytes - roundtrip(false, "\x80\xbf\x80\xbf\x80\xbf\x80"); - - // 3.1.9 Sequence of all 64 possible continuation bytes (0x80-0xbf) - roundtrip(false, "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"); - } - - SECTION("3.2 Lonely start characters") - { - // 3.2.1 All 32 first bytes of 2-byte sequences (0xc0-0xdf) - roundtrip(false, "\xc0 \xc1 \xc2 \xc3 \xc4 \xc5 \xc6 \xc7 \xc8 \xc9 \xca \xcb \xcc \xcd \xce \xcf \xd0 \xd1 \xd2 \xd3 \xd4 \xd5 \xd6 \xd7 \xd8 \xd9 \xda \xdb \xdc \xdd \xde \xdf"); - // 3.2.2 All 16 first bytes of 3-byte sequences (0xe0-0xef) - roundtrip(false, "\xe0 \xe1 \xe2 \xe3 \xe4 \xe5 \xe6 \xe7 \xe8 \xe9 \xea \xeb \xec \xed \xee \xef"); - // 3.2.3 All 8 first bytes of 4-byte sequences (0xf0-0xf7) - roundtrip(false, "\xf0 \xf1 \xf2 \xf3 \xf4 \xf5 \xf6 \xf7"); - // 3.2.4 All 4 first bytes of 5-byte sequences (0xf8-0xfb) - roundtrip(false, "\xf8 \xf9 \xfa \xfb"); - // 3.2.5 All 2 first bytes of 6-byte sequences (0xfc-0xfd) - roundtrip(false, "\xfc \xfd"); - } - - SECTION("3.3 Sequences with last continuation byte missing") - { - // All bytes of an incomplete sequence should be signalled as a single - // malformed sequence, i.e., you should see only a single replacement - // character in each of the next 10 tests. (Characters as in section 2) - - // 3.3.1 2-byte sequence with last byte missing (U+0000) - roundtrip(false, "\xc0"); - // 3.3.2 3-byte sequence with last byte missing (U+0000) - roundtrip(false, "\xe0\x80"); - // 3.3.3 4-byte sequence with last byte missing (U+0000) - roundtrip(false, "\xf0\x80\x80"); - // 3.3.4 5-byte sequence with last byte missing (U+0000) - roundtrip(false, "\xf8\x80\x80\x80"); - // 3.3.5 6-byte sequence with last byte missing (U+0000) - roundtrip(false, "\xfc\x80\x80\x80\x80"); - // 3.3.6 2-byte sequence with last byte missing (U-000007FF) - roundtrip(false, "\xdf"); - // 3.3.7 3-byte sequence with last byte missing (U-0000FFFF) - roundtrip(false, "\xef\xbf"); - // 3.3.8 4-byte sequence with last byte missing (U-001FFFFF) - roundtrip(false, "\xf7\xbf\xbf"); - // 3.3.9 5-byte sequence with last byte missing (U-03FFFFFF) - roundtrip(false, "\xfb\xbf\xbf\xbf"); - // 3.3.10 6-byte sequence with last byte missing (U-7FFFFFFF) - roundtrip(false, "\xfd\xbf\xbf\xbf\xbf"); - } - - SECTION("3.4 Concatenation of incomplete sequences") - { - // All the 10 sequences of 3.3 concatenated, you should see 10 malformed - // sequences being signalled: - roundtrip(false, "\xc0\xe0\x80\xf0\x80\x80\xf8\x80\x80\x80\xfc\x80\x80\x80\x80\xdf\xef\xbf\xf7\xbf\xbf\xfb\xbf\xbf\xbf\xfd\xbf\xbf\xbf\xbf"); - } - - SECTION("3.5 Impossible bytes") - { - // The following two bytes cannot appear in a correct UTF-8 string - - // 3.5.1 fe - roundtrip(false, "\xfe"); - // 3.5.2 ff - roundtrip(false, "\xff"); - // 3.5.3 fe fe ff ff - roundtrip(false, "\xfe\xfe\xff\xff"); - } - } - - SECTION("4 Overlong sequences") - { - // The following sequences are not malformed according to the letter of - // the Unicode 2.0 standard. However, they are longer then necessary and - // a correct UTF-8 encoder is not allowed to produce them. A "safe UTF-8 - // decoder" should reject them just like malformed sequences for two - // reasons: (1) It helps to debug applications if overlong sequences are - // not treated as valid representations of characters, because this helps - // to spot problems more quickly. (2) Overlong sequences provide - // alternative representations of characters, that could maliciously be - // used to bypass filters that check only for ASCII characters. For - // instance, a 2-byte encoded line feed (LF) would not be caught by a - // line counter that counts only 0x0a bytes, but it would still be - // processed as a line feed by an unsafe UTF-8 decoder later in the - // pipeline. From a security point of view, ASCII compatibility of UTF-8 - // sequences means also, that ASCII characters are *only* allowed to be - // represented by ASCII bytes in the range 0x00-0x7f. To ensure this - // aspect of ASCII compatibility, use only "safe UTF-8 decoders" that - // reject overlong UTF-8 sequences for which a shorter encoding exists. - - SECTION("4.1 Examples of an overlong ASCII character") - { - // With a safe UTF-8 decoder, all the following five overlong - // representations of the ASCII character slash ("/") should be rejected - // like a malformed UTF-8 sequence, for instance by substituting it with - // a replacement character. If you see a slash below, you do not have a - // safe UTF-8 decoder! - - // 4.1.1 U+002F = c0 af - roundtrip(false, "\xc0\xaf"); - // 4.1.2 U+002F = e0 80 af - roundtrip(false, "\xe0\x80\xaf"); - // 4.1.3 U+002F = f0 80 80 af - roundtrip(false, "\xf0\x80\x80\xaf"); - // 4.1.4 U+002F = f8 80 80 80 af - roundtrip(false, "\xf8\x80\x80\x80\xaf"); - // 4.1.5 U+002F = fc 80 80 80 80 af - roundtrip(false, "\xfc\x80\x80\x80\x80\xaf"); - } - - SECTION("4.2 Maximum overlong sequences") - { - // Below you see the highest Unicode value that is still resulting in an - // overlong sequence if represented with the given number of bytes. This - // is a boundary test for safe UTF-8 decoders. All five characters should - // be rejected like malformed UTF-8 sequences. - - // 4.2.1 U-0000007F = c1 bf - roundtrip(false, "\xc1\xbf"); - // 4.2.2 U-000007FF = e0 9f bf - roundtrip(false, "\xe0\x9f\xbf"); - // 4.2.3 U-0000FFFF = f0 8f bf bf - roundtrip(false, "\xf0\x8f\xbf\xbf"); - // 4.2.4 U-001FFFFF = f8 87 bf bf bf - roundtrip(false, "\xf8\x87\xbf\xbf\xbf"); - // 4.2.5 U-03FFFFFF = fc 83 bf bf bf bf - roundtrip(false, "\xfc\x83\xbf\xbf\xbf\xbf"); - } - - SECTION("4.3 Overlong representation of the NUL character") - { - // The following five sequences should also be rejected like malformed - // UTF-8 sequences and should not be treated like the ASCII NUL - // character. - - // 4.3.1 U+0000 = c0 80 - roundtrip(false, "\xc0\x80"); - // 4.3.2 U+0000 = e0 80 80 - roundtrip(false, "\xe0\x80\x80"); - // 4.3.3 U+0000 = f0 80 80 80 - roundtrip(false, "\xf0\x80\x80\x80"); - // 4.3.4 U+0000 = f8 80 80 80 80 - roundtrip(false, "\xf8\x80\x80\x80\x80"); - // 4.3.5 U+0000 = fc 80 80 80 80 80 - roundtrip(false, "\xfc\x80\x80\x80\x80\x80"); - } - } - - SECTION("5 Illegal code positions") - { - // The following UTF-8 sequences should be rejected like malformed - // sequences, because they never represent valid ISO 10646 characters and - // a UTF-8 decoder that accepts them might introduce security problems - // comparable to overlong UTF-8 sequences. - - SECTION("5.1 Single UTF-16 surrogates") - { - // 5.1.1 U+D800 = ed a0 80 - roundtrip(false, "\xed\xa0\x80"); - // 5.1.2 U+DB7F = ed ad bf - roundtrip(false, "\xed\xad\xbf"); - // 5.1.3 U+DB80 = ed ae 80 - roundtrip(false, "\xed\xae\x80"); - // 5.1.4 U+DBFF = ed af bf - roundtrip(false, "\xed\xaf\xbf"); - // 5.1.5 U+DC00 = ed b0 80 - roundtrip(false, "\xed\xb0\x80"); - // 5.1.6 U+DF80 = ed be 80 - roundtrip(false, "\xed\xbe\x80"); - // 5.1.7 U+DFFF = ed bf bf - roundtrip(false, "\xed\xbf\xbf"); - } - - SECTION("5.2 Paired UTF-16 surrogates") - { - // 5.2.1 U+D800 U+DC00 = ed a0 80 ed b0 80 - roundtrip(false, "\xed\xa0\x80\xed\xb0\x80"); - // 5.2.2 U+D800 U+DFFF = ed a0 80 ed bf bf - roundtrip(false, "\xed\xa0\x80\xed\xbf\xbf"); - // 5.2.3 U+DB7F U+DC00 = ed ad bf ed b0 80 - roundtrip(false, "\xed\xad\xbf\xed\xb0\x80"); - // 5.2.4 U+DB7F U+DFFF = ed ad bf ed bf bf - roundtrip(false, "\xed\xad\xbf\xed\xbf\xbf"); - // 5.2.5 U+DB80 U+DC00 = ed ae 80 ed b0 80 - roundtrip(false, "\xed\xae\x80\xed\xb0\x80"); - // 5.2.6 U+DB80 U+DFFF = ed ae 80 ed bf bf - roundtrip(false, "\xed\xae\x80\xed\xbf\xbf"); - // 5.2.7 U+DBFF U+DC00 = ed af bf ed b0 80 - roundtrip(false, "\xed\xaf\xbf\xed\xb0\x80"); - // 5.2.8 U+DBFF U+DFFF = ed af bf ed bf bf - roundtrip(false, "\xed\xaf\xbf\xed\xbf\xbf"); - } - - SECTION("5.3 Noncharacter code positions") - { - // The following "noncharacters" are "reserved for internal use" by - // applications, and according to older versions of the Unicode Standard - // "should never be interchanged". Unicode Corrigendum #9 dropped the - // latter restriction. Nevertheless, their presence in incoming UTF-8 data - // can remain a potential security risk, depending on what use is made of - // these codes subsequently. Examples of such internal use: - // - // - Some file APIs with 16-bit characters may use the integer value -1 - // = U+FFFF to signal an end-of-file (EOF) or error condition. - // - // - In some UTF-16 receivers, code point U+FFFE might trigger a - // byte-swap operation (to convert between UTF-16LE and UTF-16BE). - // - // With such internal use of noncharacters, it may be desirable and safer - // to block those code points in UTF-8 decoders, as they should never - // occur legitimately in incoming UTF-8 data, and could trigger unsafe - // behaviour in subsequent processing. - - // Particularly problematic noncharacters in 16-bit applications: - - // 5.3.1 U+FFFE = ef bf be - roundtrip(true, "\xef\xbf\xbe"); - // 5.3.2 U+FFFF = ef bf bf - roundtrip(true, "\xef\xbf\xbf"); - - // 5.3.3 U+FDD0 .. U+FDEF - roundtrip(true, "\xEF\xB7\x90"); - roundtrip(true, "\xEF\xB7\x91"); - roundtrip(true, "\xEF\xB7\x92"); - roundtrip(true, "\xEF\xB7\x93"); - roundtrip(true, "\xEF\xB7\x94"); - roundtrip(true, "\xEF\xB7\x95"); - roundtrip(true, "\xEF\xB7\x96"); - roundtrip(true, "\xEF\xB7\x97"); - roundtrip(true, "\xEF\xB7\x98"); - roundtrip(true, "\xEF\xB7\x99"); - roundtrip(true, "\xEF\xB7\x9A"); - roundtrip(true, "\xEF\xB7\x9B"); - roundtrip(true, "\xEF\xB7\x9C"); - roundtrip(true, "\xEF\xB7\x9D"); - roundtrip(true, "\xEF\xB7\x9E"); - roundtrip(true, "\xEF\xB7\x9F"); - roundtrip(true, "\xEF\xB7\xA0"); - roundtrip(true, "\xEF\xB7\xA1"); - roundtrip(true, "\xEF\xB7\xA2"); - roundtrip(true, "\xEF\xB7\xA3"); - roundtrip(true, "\xEF\xB7\xA4"); - roundtrip(true, "\xEF\xB7\xA5"); - roundtrip(true, "\xEF\xB7\xA6"); - roundtrip(true, "\xEF\xB7\xA7"); - roundtrip(true, "\xEF\xB7\xA8"); - roundtrip(true, "\xEF\xB7\xA9"); - roundtrip(true, "\xEF\xB7\xAA"); - roundtrip(true, "\xEF\xB7\xAB"); - roundtrip(true, "\xEF\xB7\xAC"); - roundtrip(true, "\xEF\xB7\xAD"); - roundtrip(true, "\xEF\xB7\xAE"); - roundtrip(true, "\xEF\xB7\xAF"); - - // 5.3.4 U+nFFFE U+nFFFF (for n = 1..10) - roundtrip(true, "\xF0\x9F\xBF\xBF"); - roundtrip(true, "\xF0\xAF\xBF\xBF"); - roundtrip(true, "\xF0\xBF\xBF\xBF"); - roundtrip(true, "\xF1\x8F\xBF\xBF"); - roundtrip(true, "\xF1\x9F\xBF\xBF"); - roundtrip(true, "\xF1\xAF\xBF\xBF"); - roundtrip(true, "\xF1\xBF\xBF\xBF"); - roundtrip(true, "\xF2\x8F\xBF\xBF"); - roundtrip(true, "\xF2\x9F\xBF\xBF"); - roundtrip(true, "\xF2\xAF\xBF\xBF"); - } - } -} diff --git a/tests/src/unit-unicode2.cpp b/tests/src/unit-unicode2.cpp deleted file mode 100644 index a9649b4de..000000000 --- a/tests/src/unit-unicode2.cpp +++ /dev/null @@ -1,612 +0,0 @@ -// __ _____ _____ _____ -// __| | __| | | | JSON for Modern C++ (supporting code) -// | | |__ | | | | | | version 3.12.0 -// |_____|_____|_____|_|___| https://github.com/nlohmann/json -// -// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann -// SPDX-License-Identifier: MIT - -#include "doctest_compatibility.h" - -// for some reason including this after the json header leads to linker errors with VS 2017... -#include - -#include -using nlohmann::json; - -#include -#include -#include -#include -#include "make_test_data_available.hpp" -#include "test_utils.hpp" - -// this test suite uses static variables with non-trivial destructors -DOCTEST_CLANG_SUPPRESS_WARNING_PUSH -DOCTEST_CLANG_SUPPRESS_WARNING("-Wexit-time-destructors") - -namespace -{ -extern size_t calls; -size_t calls = 0; - -void check_utf8dump(bool success_expected, int byte1, int byte2, int byte3, int byte4); - -void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3 = -1, int byte4 = -1) -{ - static std::string json_string; - json_string.clear(); - - CAPTURE(byte1) - CAPTURE(byte2) - CAPTURE(byte3) - CAPTURE(byte4) - - json_string += std::string(1, static_cast(byte1)); - - if (byte2 != -1) - { - json_string += std::string(1, static_cast(byte2)); - } - - if (byte3 != -1) - { - json_string += std::string(1, static_cast(byte3)); - } - - if (byte4 != -1) - { - json_string += std::string(1, static_cast(byte4)); - } - - CAPTURE(json_string) - - // store the string in a JSON value - static json j; - static json j2; - j = json_string; - j2 = "abc" + json_string + "xyz"; - - static std::string s_ignored; - static std::string s_ignored2; - static std::string s_ignored_ascii; - static std::string s_ignored2_ascii; - static std::string s_replaced; - static std::string s_replaced2; - static std::string s_replaced_ascii; - static std::string s_replaced2_ascii; - - // dumping with ignore/replace must not throw in any case - s_ignored = j.dump(-1, ' ', false, json::error_handler_t::ignore); - s_ignored2 = j2.dump(-1, ' ', false, json::error_handler_t::ignore); - s_ignored_ascii = j.dump(-1, ' ', true, json::error_handler_t::ignore); - s_ignored2_ascii = j2.dump(-1, ' ', true, json::error_handler_t::ignore); - s_replaced = j.dump(-1, ' ', false, json::error_handler_t::replace); - s_replaced2 = j2.dump(-1, ' ', false, json::error_handler_t::replace); - s_replaced_ascii = j.dump(-1, ' ', true, json::error_handler_t::replace); - s_replaced2_ascii = j2.dump(-1, ' ', true, json::error_handler_t::replace); - - if (success_expected) - { - static std::string s_strict; - // strict mode must not throw if success is expected - s_strict = j.dump(); - // all dumps should agree on the string - CHECK(s_strict == s_ignored); - CHECK(s_strict == s_replaced); - } - else - { - // strict mode must throw if success is not expected - // dump() is nodiscard; the exception is thrown by dump() itself before it would return - CHECK_THROWS_AS(utils::ignore_return_value(j.dump()), json::type_error&); - // ignore and replace must create different dumps - CHECK(s_ignored != s_replaced); - - // check that replace string contains a replacement character - CHECK(s_replaced.find("\xEF\xBF\xBD") != std::string::npos); - } - - // check that prefix and suffix are preserved - CHECK(s_ignored2.substr(1, 3) == "abc"); - CHECK(s_ignored2.substr(s_ignored2.size() - 4, 3) == "xyz"); - CHECK(s_ignored2_ascii.substr(1, 3) == "abc"); - CHECK(s_ignored2_ascii.substr(s_ignored2_ascii.size() - 4, 3) == "xyz"); - CHECK(s_replaced2.substr(1, 3) == "abc"); - CHECK(s_replaced2.substr(s_replaced2.size() - 4, 3) == "xyz"); - CHECK(s_replaced2_ascii.substr(1, 3) == "abc"); - CHECK(s_replaced2_ascii.substr(s_replaced2_ascii.size() - 4, 3) == "xyz"); -} - -void check_utf8string(bool success_expected, int byte1, int byte2, int byte3, int byte4); - -// create and check a JSON string with up to four UTF-8 bytes -void check_utf8string(bool success_expected, int byte1, int byte2 = -1, int byte3 = -1, int byte4 = -1) -{ - if (++calls % 100000 == 0) - { - std::cout << calls << " of 455355 UTF-8 strings checked" << std::endl; // NOLINT(performance-avoid-endl) - } - - static std::string json_string; - json_string = "\""; - - CAPTURE(byte1) - json_string += std::string(1, static_cast(byte1)); - - if (byte2 != -1) - { - CAPTURE(byte2) - json_string += std::string(1, static_cast(byte2)); - } - - if (byte3 != -1) - { - CAPTURE(byte3) - json_string += std::string(1, static_cast(byte3)); - } - - if (byte4 != -1) - { - CAPTURE(byte4) - json_string += std::string(1, static_cast(byte4)); - } - - json_string += "\""; - - CAPTURE(json_string) - - json _; - if (success_expected) - { - CHECK_NOTHROW(_ = json::parse(json_string)); - } - else - { - CHECK_THROWS_AS(_ = json::parse(json_string), json::parse_error&); - } -} -} // namespace - -TEST_CASE("Unicode (2/5)" * doctest::skip()) -{ - SECTION("RFC 3629") - { - /* - RFC 3629 describes in Sect. 4 the syntax of UTF-8 byte sequences as - follows: - - A UTF-8 string is a sequence of octets representing a sequence of UCS - characters. An octet sequence is valid UTF-8 only if it matches the - following syntax, which is derived from the rules for encoding UTF-8 - and is expressed in the ABNF of [RFC2234]. - - UTF8-octets = *( UTF8-char ) - UTF8-char = UTF8-1 / UTF8-2 / UTF8-3 / UTF8-4 - UTF8-1 = %x00-7F - UTF8-2 = %xC2-DF UTF8-tail - UTF8-3 = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2( UTF8-tail ) / - %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail ) - UTF8-4 = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) / - %xF4 %x80-8F 2( UTF8-tail ) - UTF8-tail = %x80-BF - */ - - SECTION("ill-formed first byte") - { - for (int byte1 = 0x80; byte1 <= 0xC1; ++byte1) - { - check_utf8string(false, byte1); - check_utf8dump(false, byte1); - } - - for (int byte1 = 0xF5; byte1 <= 0xFF; ++byte1) - { - check_utf8string(false, byte1); - check_utf8dump(false, byte1); - } - } - - SECTION("UTF8-1 (x00-x7F)") - { - SECTION("well-formed") - { - for (int byte1 = 0x00; byte1 <= 0x7F; ++byte1) - { - // unescaped control characters are parse errors in JSON - if (0x00 <= byte1 && byte1 <= 0x1F) - { - check_utf8string(false, byte1); - continue; - } - - // a single quote is a parse error in JSON - if (byte1 == 0x22) - { - check_utf8string(false, byte1); - continue; - } - - // a single backslash is a parse error in JSON - if (byte1 == 0x5C) - { - check_utf8string(false, byte1); - continue; - } - - // all other characters are OK - check_utf8string(true, byte1); - check_utf8dump(true, byte1); - } - } - } - - SECTION("UTF8-2 (xC2-xDF UTF8-tail)") - { - SECTION("well-formed") - { - for (int byte1 = 0xC2; byte1 <= 0xDF; ++byte1) - { - for (int byte2 = 0x80; byte2 <= 0xBF; ++byte2) - { - check_utf8string(true, byte1, byte2); - check_utf8dump(true, byte1, byte2); - } - } - } - - SECTION("ill-formed: missing second byte") - { - for (int byte1 = 0xC2; byte1 <= 0xDF; ++byte1) - { - check_utf8string(false, byte1); - check_utf8dump(false, byte1); - } - } - - SECTION("ill-formed: wrong second byte") - { - for (int byte1 = 0xC2; byte1 <= 0xDF; ++byte1) - { - for (int byte2 = 0x00; byte2 <= 0xFF; ++byte2) - { - // skip correct second byte - if (0x80 <= byte2 && byte2 <= 0xBF) - { - continue; - } - - check_utf8string(false, byte1, byte2); - check_utf8dump(false, byte1, byte2); - } - } - } - } - - SECTION("UTF8-3 (xE0 xA0-BF UTF8-tail)") - { - SECTION("well-formed") - { - for (int byte1 = 0xE0; byte1 <= 0xE0; ++byte1) - { - for (int byte2 = 0xA0; byte2 <= 0xBF; ++byte2) - { - for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3) - { - check_utf8string(true, byte1, byte2, byte3); - check_utf8dump(true, byte1, byte2, byte3); - } - } - } - } - - SECTION("ill-formed: missing second byte") - { - for (int byte1 = 0xE0; byte1 <= 0xE0; ++byte1) - { - check_utf8string(false, byte1); - check_utf8dump(false, byte1); - } - } - - SECTION("ill-formed: missing third byte") - { - for (int byte1 = 0xE0; byte1 <= 0xE0; ++byte1) - { - for (int byte2 = 0xA0; byte2 <= 0xBF; ++byte2) - { - check_utf8string(false, byte1, byte2); - check_utf8dump(false, byte1, byte2); - } - } - } - - SECTION("ill-formed: wrong second byte") - { - for (int byte1 = 0xE0; byte1 <= 0xE0; ++byte1) - { - for (int byte2 = 0x00; byte2 <= 0xFF; ++byte2) - { - // skip correct second byte - if (0xA0 <= byte2 && byte2 <= 0xBF) - { - continue; - } - - for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3) - { - check_utf8string(false, byte1, byte2, byte3); - check_utf8dump(false, byte1, byte2, byte3); - } - } - } - } - - SECTION("ill-formed: wrong third byte") - { - for (int byte1 = 0xE0; byte1 <= 0xE0; ++byte1) - { - for (int byte2 = 0xA0; byte2 <= 0xBF; ++byte2) - { - for (int byte3 = 0x00; byte3 <= 0xFF; ++byte3) - { - // skip correct third byte - if (0x80 <= byte3 && byte3 <= 0xBF) - { - continue; - } - - check_utf8string(false, byte1, byte2, byte3); - check_utf8dump(false, byte1, byte2, byte3); - } - } - } - } - } - - SECTION("UTF8-3 (xE1-xEC UTF8-tail UTF8-tail)") - { - SECTION("well-formed") - { - for (int byte1 = 0xE1; byte1 <= 0xEC; ++byte1) - { - for (int byte2 = 0x80; byte2 <= 0xBF; ++byte2) - { - for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3) - { - check_utf8string(true, byte1, byte2, byte3); - check_utf8dump(true, byte1, byte2, byte3); - } - } - } - } - - SECTION("ill-formed: missing second byte") - { - for (int byte1 = 0xE1; byte1 <= 0xEC; ++byte1) - { - check_utf8string(false, byte1); - check_utf8dump(false, byte1); - } - } - - SECTION("ill-formed: missing third byte") - { - for (int byte1 = 0xE1; byte1 <= 0xEC; ++byte1) - { - for (int byte2 = 0x80; byte2 <= 0xBF; ++byte2) - { - check_utf8string(false, byte1, byte2); - check_utf8dump(false, byte1, byte2); - } - } - } - - SECTION("ill-formed: wrong second byte") - { - for (int byte1 = 0xE1; byte1 <= 0xEC; ++byte1) - { - for (int byte2 = 0x00; byte2 <= 0xFF; ++byte2) - { - // skip correct second byte - if (0x80 <= byte2 && byte2 <= 0xBF) - { - continue; - } - - for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3) - { - check_utf8string(false, byte1, byte2, byte3); - check_utf8dump(false, byte1, byte2, byte3); - } - } - } - } - - SECTION("ill-formed: wrong third byte") - { - for (int byte1 = 0xE1; byte1 <= 0xEC; ++byte1) - { - for (int byte2 = 0x80; byte2 <= 0xBF; ++byte2) - { - for (int byte3 = 0x00; byte3 <= 0xFF; ++byte3) - { - // skip correct third byte - if (0x80 <= byte3 && byte3 <= 0xBF) - { - continue; - } - - check_utf8string(false, byte1, byte2, byte3); - check_utf8dump(false, byte1, byte2, byte3); - } - } - } - } - } - - SECTION("UTF8-3 (xED x80-9F UTF8-tail)") - { - SECTION("well-formed") - { - for (int byte1 = 0xED; byte1 <= 0xED; ++byte1) - { - for (int byte2 = 0x80; byte2 <= 0x9F; ++byte2) - { - for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3) - { - check_utf8string(true, byte1, byte2, byte3); - check_utf8dump(true, byte1, byte2, byte3); - } - } - } - } - - SECTION("ill-formed: missing second byte") - { - for (int byte1 = 0xED; byte1 <= 0xED; ++byte1) - { - check_utf8string(false, byte1); - check_utf8dump(false, byte1); - } - } - - SECTION("ill-formed: missing third byte") - { - for (int byte1 = 0xED; byte1 <= 0xED; ++byte1) - { - for (int byte2 = 0x80; byte2 <= 0x9F; ++byte2) - { - check_utf8string(false, byte1, byte2); - check_utf8dump(false, byte1, byte2); - } - } - } - - SECTION("ill-formed: wrong second byte") - { - for (int byte1 = 0xED; byte1 <= 0xED; ++byte1) - { - for (int byte2 = 0x00; byte2 <= 0xFF; ++byte2) - { - // skip correct second byte - if (0x80 <= byte2 && byte2 <= 0x9F) - { - continue; - } - - for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3) - { - check_utf8string(false, byte1, byte2, byte3); - check_utf8dump(false, byte1, byte2, byte3); - } - } - } - } - - SECTION("ill-formed: wrong third byte") - { - for (int byte1 = 0xED; byte1 <= 0xED; ++byte1) - { - for (int byte2 = 0x80; byte2 <= 0x9F; ++byte2) - { - for (int byte3 = 0x00; byte3 <= 0xFF; ++byte3) - { - // skip correct third byte - if (0x80 <= byte3 && byte3 <= 0xBF) - { - continue; - } - - check_utf8string(false, byte1, byte2, byte3); - check_utf8dump(false, byte1, byte2, byte3); - } - } - } - } - } - - SECTION("UTF8-3 (xEE-xEF UTF8-tail UTF8-tail)") - { - SECTION("well-formed") - { - for (int byte1 = 0xEE; byte1 <= 0xEF; ++byte1) - { - for (int byte2 = 0x80; byte2 <= 0xBF; ++byte2) - { - for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3) - { - check_utf8string(true, byte1, byte2, byte3); - check_utf8dump(true, byte1, byte2, byte3); - } - } - } - } - - SECTION("ill-formed: missing second byte") - { - for (int byte1 = 0xEE; byte1 <= 0xEF; ++byte1) - { - check_utf8string(false, byte1); - check_utf8dump(false, byte1); - } - } - - SECTION("ill-formed: missing third byte") - { - for (int byte1 = 0xEE; byte1 <= 0xEF; ++byte1) - { - for (int byte2 = 0x80; byte2 <= 0xBF; ++byte2) - { - check_utf8string(false, byte1, byte2); - check_utf8dump(false, byte1, byte2); - } - } - } - - SECTION("ill-formed: wrong second byte") - { - for (int byte1 = 0xEE; byte1 <= 0xEF; ++byte1) - { - for (int byte2 = 0x00; byte2 <= 0xFF; ++byte2) - { - // skip correct second byte - if (0x80 <= byte2 && byte2 <= 0xBF) - { - continue; - } - - for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3) - { - check_utf8string(false, byte1, byte2, byte3); - check_utf8dump(false, byte1, byte2, byte3); - } - } - } - } - - SECTION("ill-formed: wrong third byte") - { - for (int byte1 = 0xEE; byte1 <= 0xEF; ++byte1) - { - for (int byte2 = 0x80; byte2 <= 0xBF; ++byte2) - { - for (int byte3 = 0x00; byte3 <= 0xFF; ++byte3) - { - // skip correct third byte - if (0x80 <= byte3 && byte3 <= 0xBF) - { - continue; - } - - check_utf8string(false, byte1, byte2, byte3); - check_utf8dump(false, byte1, byte2, byte3); - } - } - } - } - } - } -} - -DOCTEST_CLANG_SUPPRESS_WARNING_POP diff --git a/tests/src/unit-unicode3.cpp b/tests/src/unit-unicode3.cpp deleted file mode 100644 index 12c12eea4..000000000 --- a/tests/src/unit-unicode3.cpp +++ /dev/null @@ -1,326 +0,0 @@ -// __ _____ _____ _____ -// __| | __| | | | JSON for Modern C++ (supporting code) -// | | |__ | | | | | | version 3.12.0 -// |_____|_____|_____|_|___| https://github.com/nlohmann/json -// -// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann -// SPDX-License-Identifier: MIT - -#include "doctest_compatibility.h" - -// for some reason including this after the json header leads to linker errors with VS 2017... -#include - -#include -using nlohmann::json; - -#include -#include -#include -#include -#include "make_test_data_available.hpp" -#include "test_utils.hpp" - -// this test suite uses static variables with non-trivial destructors -DOCTEST_CLANG_SUPPRESS_WARNING_PUSH -DOCTEST_CLANG_SUPPRESS_WARNING("-Wexit-time-destructors") - -namespace -{ -extern size_t calls; -size_t calls = 0; - -void check_utf8dump(bool success_expected, int byte1, int byte2, int byte3, int byte4); - -void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3 = -1, int byte4 = -1) -{ - static std::string json_string; - json_string.clear(); - - CAPTURE(byte1) - CAPTURE(byte2) - CAPTURE(byte3) - CAPTURE(byte4) - - json_string += std::string(1, static_cast(byte1)); - - if (byte2 != -1) - { - json_string += std::string(1, static_cast(byte2)); - } - - if (byte3 != -1) - { - json_string += std::string(1, static_cast(byte3)); - } - - if (byte4 != -1) - { - json_string += std::string(1, static_cast(byte4)); - } - - CAPTURE(json_string) - - // store the string in a JSON value - static json j; - static json j2; - j = json_string; - j2 = "abc" + json_string + "xyz"; - - static std::string s_ignored; - static std::string s_ignored2; - static std::string s_ignored_ascii; - static std::string s_ignored2_ascii; - static std::string s_replaced; - static std::string s_replaced2; - static std::string s_replaced_ascii; - static std::string s_replaced2_ascii; - - // dumping with ignore/replace must not throw in any case - s_ignored = j.dump(-1, ' ', false, json::error_handler_t::ignore); - s_ignored2 = j2.dump(-1, ' ', false, json::error_handler_t::ignore); - s_ignored_ascii = j.dump(-1, ' ', true, json::error_handler_t::ignore); - s_ignored2_ascii = j2.dump(-1, ' ', true, json::error_handler_t::ignore); - s_replaced = j.dump(-1, ' ', false, json::error_handler_t::replace); - s_replaced2 = j2.dump(-1, ' ', false, json::error_handler_t::replace); - s_replaced_ascii = j.dump(-1, ' ', true, json::error_handler_t::replace); - s_replaced2_ascii = j2.dump(-1, ' ', true, json::error_handler_t::replace); - - if (success_expected) - { - static std::string s_strict; - // strict mode must not throw if success is expected - s_strict = j.dump(); - // all dumps should agree on the string - CHECK(s_strict == s_ignored); - CHECK(s_strict == s_replaced); - } - else - { - // strict mode must throw if success is not expected - // dump() is nodiscard; the exception is thrown by dump() itself before it would return - CHECK_THROWS_AS(utils::ignore_return_value(j.dump()), json::type_error&); - // ignore and replace must create different dumps - CHECK(s_ignored != s_replaced); - - // check that replace string contains a replacement character - CHECK(s_replaced.find("\xEF\xBF\xBD") != std::string::npos); - } - - // check that prefix and suffix are preserved - CHECK(s_ignored2.substr(1, 3) == "abc"); - CHECK(s_ignored2.substr(s_ignored2.size() - 4, 3) == "xyz"); - CHECK(s_ignored2_ascii.substr(1, 3) == "abc"); - CHECK(s_ignored2_ascii.substr(s_ignored2_ascii.size() - 4, 3) == "xyz"); - CHECK(s_replaced2.substr(1, 3) == "abc"); - CHECK(s_replaced2.substr(s_replaced2.size() - 4, 3) == "xyz"); - CHECK(s_replaced2_ascii.substr(1, 3) == "abc"); - CHECK(s_replaced2_ascii.substr(s_replaced2_ascii.size() - 4, 3) == "xyz"); -} - -void check_utf8string(bool success_expected, int byte1, int byte2, int byte3, int byte4); - -// create and check a JSON string with up to four UTF-8 bytes -void check_utf8string(bool success_expected, int byte1, int byte2 = -1, int byte3 = -1, int byte4 = -1) -{ - if (++calls % 100000 == 0) - { - std::cout << calls << " of 1641521 UTF-8 strings checked" << std::endl; // NOLINT(performance-avoid-endl) - } - - static std::string json_string; - json_string = "\""; - - CAPTURE(byte1) - json_string += std::string(1, static_cast(byte1)); - - if (byte2 != -1) - { - CAPTURE(byte2) - json_string += std::string(1, static_cast(byte2)); - } - - if (byte3 != -1) - { - CAPTURE(byte3) - json_string += std::string(1, static_cast(byte3)); - } - - if (byte4 != -1) - { - CAPTURE(byte4) - json_string += std::string(1, static_cast(byte4)); - } - - json_string += "\""; - - CAPTURE(json_string) - - json _; - if (success_expected) - { - CHECK_NOTHROW(_ = json::parse(json_string)); - } - else - { - CHECK_THROWS_AS(_ = json::parse(json_string), json::parse_error&); - } -} -} // namespace - -TEST_CASE("Unicode (3/5)" * doctest::skip()) -{ - SECTION("RFC 3629") - { - /* - RFC 3629 describes in Sect. 4 the syntax of UTF-8 byte sequences as - follows: - - A UTF-8 string is a sequence of octets representing a sequence of UCS - characters. An octet sequence is valid UTF-8 only if it matches the - following syntax, which is derived from the rules for encoding UTF-8 - and is expressed in the ABNF of [RFC2234]. - - UTF8-octets = *( UTF8-char ) - UTF8-char = UTF8-1 / UTF8-2 / UTF8-3 / UTF8-4 - UTF8-1 = %x00-7F - UTF8-2 = %xC2-DF UTF8-tail - UTF8-3 = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2( UTF8-tail ) / - %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail ) - UTF8-4 = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) / - %xF4 %x80-8F 2( UTF8-tail ) - UTF8-tail = %x80-BF - */ - - SECTION("UTF8-4 (xF0 x90-BF UTF8-tail UTF8-tail)") - { - SECTION("well-formed") - { - for (int byte1 = 0xF0; byte1 <= 0xF0; ++byte1) - { - for (int byte2 = 0x90; byte2 <= 0xBF; ++byte2) - { - for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3) - { - for (int byte4 = 0x80; byte4 <= 0xBF; ++byte4) - { - check_utf8string(true, byte1, byte2, byte3, byte4); - check_utf8dump(true, byte1, byte2, byte3, byte4); - } - } - } - } - } - - SECTION("ill-formed: missing second byte") - { - for (int byte1 = 0xF0; byte1 <= 0xF0; ++byte1) - { - check_utf8string(false, byte1); - check_utf8dump(false, byte1); - } - } - - SECTION("ill-formed: missing third byte") - { - for (int byte1 = 0xF0; byte1 <= 0xF0; ++byte1) - { - for (int byte2 = 0x90; byte2 <= 0xBF; ++byte2) - { - check_utf8string(false, byte1, byte2); - check_utf8dump(false, byte1, byte2); - } - } - } - - SECTION("ill-formed: missing fourth byte") - { - for (int byte1 = 0xF0; byte1 <= 0xF0; ++byte1) - { - for (int byte2 = 0x90; byte2 <= 0xBF; ++byte2) - { - for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3) - { - check_utf8string(false, byte1, byte2, byte3); - check_utf8dump(false, byte1, byte2, byte3); - } - } - } - } - - SECTION("ill-formed: wrong second byte") - { - for (int byte1 = 0xF0; byte1 <= 0xF0; ++byte1) - { - for (int byte2 = 0x00; byte2 <= 0xFF; ++byte2) - { - // skip correct second byte - if (0x90 <= byte2 && byte2 <= 0xBF) - { - continue; - } - - for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3) - { - for (int byte4 = 0x80; byte4 <= 0xBF; ++byte4) - { - check_utf8string(false, byte1, byte2, byte3, byte4); - check_utf8dump(false, byte1, byte2, byte3, byte4); - } - } - } - } - } - - SECTION("ill-formed: wrong third byte") - { - for (int byte1 = 0xF0; byte1 <= 0xF0; ++byte1) - { - for (int byte2 = 0x90; byte2 <= 0xBF; ++byte2) - { - for (int byte3 = 0x00; byte3 <= 0xFF; ++byte3) - { - // skip correct third byte - if (0x80 <= byte3 && byte3 <= 0xBF) - { - continue; - } - - for (int byte4 = 0x80; byte4 <= 0xBF; ++byte4) - { - check_utf8string(false, byte1, byte2, byte3, byte4); - check_utf8dump(false, byte1, byte2, byte3, byte4); - } - } - } - } - } - - SECTION("ill-formed: wrong fourth byte") - { - for (int byte1 = 0xF0; byte1 <= 0xF0; ++byte1) - { - for (int byte2 = 0x90; byte2 <= 0xBF; ++byte2) - { - for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3) - { - for (int byte4 = 0x00; byte4 <= 0xFF; ++byte4) - { - // skip correct fourth byte - if (0x80 <= byte4 && byte4 <= 0xBF) - { - continue; - } - - check_utf8string(false, byte1, byte2, byte3, byte4); - check_utf8dump(false, byte1, byte2, byte3, byte4); - } - } - } - } - } - } - } -} - -DOCTEST_CLANG_SUPPRESS_WARNING_POP diff --git a/tests/src/unit-unicode4.cpp b/tests/src/unit-unicode4.cpp deleted file mode 100644 index 43cf7095e..000000000 --- a/tests/src/unit-unicode4.cpp +++ /dev/null @@ -1,326 +0,0 @@ -// __ _____ _____ _____ -// __| | __| | | | JSON for Modern C++ (supporting code) -// | | |__ | | | | | | version 3.12.0 -// |_____|_____|_____|_|___| https://github.com/nlohmann/json -// -// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann -// SPDX-License-Identifier: MIT - -#include "doctest_compatibility.h" - -// for some reason including this after the json header leads to linker errors with VS 2017... -#include - -#include -using nlohmann::json; - -#include -#include -#include -#include -#include "make_test_data_available.hpp" -#include "test_utils.hpp" - -// this test suite uses static variables with non-trivial destructors -DOCTEST_CLANG_SUPPRESS_WARNING_PUSH -DOCTEST_CLANG_SUPPRESS_WARNING("-Wexit-time-destructors") - -namespace -{ -extern size_t calls; -size_t calls = 0; - -void check_utf8dump(bool success_expected, int byte1, int byte2, int byte3, int byte4); - -void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3 = -1, int byte4 = -1) -{ - static std::string json_string; - json_string.clear(); - - CAPTURE(byte1) - CAPTURE(byte2) - CAPTURE(byte3) - CAPTURE(byte4) - - json_string += std::string(1, static_cast(byte1)); - - if (byte2 != -1) - { - json_string += std::string(1, static_cast(byte2)); - } - - if (byte3 != -1) - { - json_string += std::string(1, static_cast(byte3)); - } - - if (byte4 != -1) - { - json_string += std::string(1, static_cast(byte4)); - } - - CAPTURE(json_string) - - // store the string in a JSON value - static json j; - static json j2; - j = json_string; - j2 = "abc" + json_string + "xyz"; - - static std::string s_ignored; - static std::string s_ignored2; - static std::string s_ignored_ascii; - static std::string s_ignored2_ascii; - static std::string s_replaced; - static std::string s_replaced2; - static std::string s_replaced_ascii; - static std::string s_replaced2_ascii; - - // dumping with ignore/replace must not throw in any case - s_ignored = j.dump(-1, ' ', false, json::error_handler_t::ignore); - s_ignored2 = j2.dump(-1, ' ', false, json::error_handler_t::ignore); - s_ignored_ascii = j.dump(-1, ' ', true, json::error_handler_t::ignore); - s_ignored2_ascii = j2.dump(-1, ' ', true, json::error_handler_t::ignore); - s_replaced = j.dump(-1, ' ', false, json::error_handler_t::replace); - s_replaced2 = j2.dump(-1, ' ', false, json::error_handler_t::replace); - s_replaced_ascii = j.dump(-1, ' ', true, json::error_handler_t::replace); - s_replaced2_ascii = j2.dump(-1, ' ', true, json::error_handler_t::replace); - - if (success_expected) - { - static std::string s_strict; - // strict mode must not throw if success is expected - s_strict = j.dump(); - // all dumps should agree on the string - CHECK(s_strict == s_ignored); - CHECK(s_strict == s_replaced); - } - else - { - // strict mode must throw if success is not expected - // dump() is nodiscard; the exception is thrown by dump() itself before it would return - CHECK_THROWS_AS(utils::ignore_return_value(j.dump()), json::type_error&); - // ignore and replace must create different dumps - CHECK(s_ignored != s_replaced); - - // check that replace string contains a replacement character - CHECK(s_replaced.find("\xEF\xBF\xBD") != std::string::npos); - } - - // check that prefix and suffix are preserved - CHECK(s_ignored2.substr(1, 3) == "abc"); - CHECK(s_ignored2.substr(s_ignored2.size() - 4, 3) == "xyz"); - CHECK(s_ignored2_ascii.substr(1, 3) == "abc"); - CHECK(s_ignored2_ascii.substr(s_ignored2_ascii.size() - 4, 3) == "xyz"); - CHECK(s_replaced2.substr(1, 3) == "abc"); - CHECK(s_replaced2.substr(s_replaced2.size() - 4, 3) == "xyz"); - CHECK(s_replaced2_ascii.substr(1, 3) == "abc"); - CHECK(s_replaced2_ascii.substr(s_replaced2_ascii.size() - 4, 3) == "xyz"); -} - -void check_utf8string(bool success_expected, int byte1, int byte2, int byte3, int byte4); - -// create and check a JSON string with up to four UTF-8 bytes -void check_utf8string(bool success_expected, int byte1, int byte2 = -1, int byte3 = -1, int byte4 = -1) -{ - if (++calls % 100000 == 0) - { - std::cout << calls << " of 5517507 UTF-8 strings checked" << std::endl; // NOLINT(performance-avoid-endl) - } - - static std::string json_string; - json_string = "\""; - - CAPTURE(byte1) - json_string += std::string(1, static_cast(byte1)); - - if (byte2 != -1) - { - CAPTURE(byte2) - json_string += std::string(1, static_cast(byte2)); - } - - if (byte3 != -1) - { - CAPTURE(byte3) - json_string += std::string(1, static_cast(byte3)); - } - - if (byte4 != -1) - { - CAPTURE(byte4) - json_string += std::string(1, static_cast(byte4)); - } - - json_string += "\""; - - CAPTURE(json_string) - - json _; - if (success_expected) - { - CHECK_NOTHROW(_ = json::parse(json_string)); - } - else - { - CHECK_THROWS_AS(_ = json::parse(json_string), json::parse_error&); - } -} -} // namespace - -TEST_CASE("Unicode (4/5)" * doctest::skip()) -{ - SECTION("RFC 3629") - { - /* - RFC 3629 describes in Sect. 4 the syntax of UTF-8 byte sequences as - follows: - - A UTF-8 string is a sequence of octets representing a sequence of UCS - characters. An octet sequence is valid UTF-8 only if it matches the - following syntax, which is derived from the rules for encoding UTF-8 - and is expressed in the ABNF of [RFC2234]. - - UTF8-octets = *( UTF8-char ) - UTF8-char = UTF8-1 / UTF8-2 / UTF8-3 / UTF8-4 - UTF8-1 = %x00-7F - UTF8-2 = %xC2-DF UTF8-tail - UTF8-3 = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2( UTF8-tail ) / - %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail ) - UTF8-4 = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) / - %xF4 %x80-8F 2( UTF8-tail ) - UTF8-tail = %x80-BF - */ - - SECTION("UTF8-4 (xF1-F3 UTF8-tail UTF8-tail UTF8-tail)") - { - SECTION("well-formed") - { - for (int byte1 = 0xF1; byte1 <= 0xF3; ++byte1) - { - for (int byte2 = 0x80; byte2 <= 0xBF; ++byte2) - { - for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3) - { - for (int byte4 = 0x80; byte4 <= 0xBF; ++byte4) - { - check_utf8string(true, byte1, byte2, byte3, byte4); - check_utf8dump(true, byte1, byte2, byte3, byte4); - } - } - } - } - } - - SECTION("ill-formed: missing second byte") - { - for (int byte1 = 0xF1; byte1 <= 0xF3; ++byte1) - { - check_utf8string(false, byte1); - check_utf8dump(false, byte1); - } - } - - SECTION("ill-formed: missing third byte") - { - for (int byte1 = 0xF1; byte1 <= 0xF3; ++byte1) - { - for (int byte2 = 0x80; byte2 <= 0xBF; ++byte2) - { - check_utf8string(false, byte1, byte2); - check_utf8dump(false, byte1, byte2); - } - } - } - - SECTION("ill-formed: missing fourth byte") - { - for (int byte1 = 0xF1; byte1 <= 0xF3; ++byte1) - { - for (int byte2 = 0x80; byte2 <= 0xBF; ++byte2) - { - for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3) - { - check_utf8string(false, byte1, byte2, byte3); - check_utf8dump(false, byte1, byte2, byte3); - } - } - } - } - - SECTION("ill-formed: wrong second byte") - { - for (int byte1 = 0xF1; byte1 <= 0xF3; ++byte1) - { - for (int byte2 = 0x00; byte2 <= 0xFF; ++byte2) - { - // skip correct second byte - if (0x80 <= byte2 && byte2 <= 0xBF) - { - continue; - } - - for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3) - { - for (int byte4 = 0x80; byte4 <= 0xBF; ++byte4) - { - check_utf8string(false, byte1, byte2, byte3, byte4); - check_utf8dump(false, byte1, byte2, byte3, byte4); - } - } - } - } - } - - SECTION("ill-formed: wrong third byte") - { - for (int byte1 = 0xF1; byte1 <= 0xF3; ++byte1) - { - for (int byte2 = 0x80; byte2 <= 0xBF; ++byte2) - { - for (int byte3 = 0x00; byte3 <= 0xFF; ++byte3) - { - // skip correct third byte - if (0x80 <= byte3 && byte3 <= 0xBF) - { - continue; - } - - for (int byte4 = 0x80; byte4 <= 0xBF; ++byte4) - { - check_utf8string(false, byte1, byte2, byte3, byte4); - check_utf8dump(false, byte1, byte2, byte3, byte4); - } - } - } - } - } - - SECTION("ill-formed: wrong fourth byte") - { - for (int byte1 = 0xF1; byte1 <= 0xF3; ++byte1) - { - for (int byte2 = 0x80; byte2 <= 0xBF; ++byte2) - { - for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3) - { - for (int byte4 = 0x00; byte4 <= 0xFF; ++byte4) - { - // skip correct fourth byte - if (0x80 <= byte4 && byte4 <= 0xBF) - { - continue; - } - - check_utf8string(false, byte1, byte2, byte3, byte4); - check_utf8dump(false, byte1, byte2, byte3, byte4); - } - } - } - } - } - } - } -} - -DOCTEST_CLANG_SUPPRESS_WARNING_POP diff --git a/tests/src/unit-unicode5.cpp b/tests/src/unit-unicode5.cpp deleted file mode 100644 index bc0312820..000000000 --- a/tests/src/unit-unicode5.cpp +++ /dev/null @@ -1,326 +0,0 @@ -// __ _____ _____ _____ -// __| | __| | | | JSON for Modern C++ (supporting code) -// | | |__ | | | | | | version 3.12.0 -// |_____|_____|_____|_|___| https://github.com/nlohmann/json -// -// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann -// SPDX-License-Identifier: MIT - -#include "doctest_compatibility.h" - -// for some reason including this after the json header leads to linker errors with VS 2017... -#include - -#include -using nlohmann::json; - -#include -#include -#include -#include -#include "make_test_data_available.hpp" -#include "test_utils.hpp" - -// this test suite uses static variables with non-trivial destructors -DOCTEST_CLANG_SUPPRESS_WARNING_PUSH -DOCTEST_CLANG_SUPPRESS_WARNING("-Wexit-time-destructors") - -namespace -{ -extern size_t calls; -size_t calls = 0; - -void check_utf8dump(bool success_expected, int byte1, int byte2, int byte3, int byte4); - -void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3 = -1, int byte4 = -1) -{ - static std::string json_string; - json_string.clear(); - - CAPTURE(byte1) - CAPTURE(byte2) - CAPTURE(byte3) - CAPTURE(byte4) - - json_string += std::string(1, static_cast(byte1)); - - if (byte2 != -1) - { - json_string += std::string(1, static_cast(byte2)); - } - - if (byte3 != -1) - { - json_string += std::string(1, static_cast(byte3)); - } - - if (byte4 != -1) - { - json_string += std::string(1, static_cast(byte4)); - } - - CAPTURE(json_string) - - // store the string in a JSON value - static json j; - static json j2; - j = json_string; - j2 = "abc" + json_string + "xyz"; - - static std::string s_ignored; - static std::string s_ignored2; - static std::string s_ignored_ascii; - static std::string s_ignored2_ascii; - static std::string s_replaced; - static std::string s_replaced2; - static std::string s_replaced_ascii; - static std::string s_replaced2_ascii; - - // dumping with ignore/replace must not throw in any case - s_ignored = j.dump(-1, ' ', false, json::error_handler_t::ignore); - s_ignored2 = j2.dump(-1, ' ', false, json::error_handler_t::ignore); - s_ignored_ascii = j.dump(-1, ' ', true, json::error_handler_t::ignore); - s_ignored2_ascii = j2.dump(-1, ' ', true, json::error_handler_t::ignore); - s_replaced = j.dump(-1, ' ', false, json::error_handler_t::replace); - s_replaced2 = j2.dump(-1, ' ', false, json::error_handler_t::replace); - s_replaced_ascii = j.dump(-1, ' ', true, json::error_handler_t::replace); - s_replaced2_ascii = j2.dump(-1, ' ', true, json::error_handler_t::replace); - - if (success_expected) - { - static std::string s_strict; - // strict mode must not throw if success is expected - s_strict = j.dump(); - // all dumps should agree on the string - CHECK(s_strict == s_ignored); - CHECK(s_strict == s_replaced); - } - else - { - // strict mode must throw if success is not expected - // dump() is nodiscard; the exception is thrown by dump() itself before it would return - CHECK_THROWS_AS(utils::ignore_return_value(j.dump()), json::type_error&); - // ignore and replace must create different dumps - CHECK(s_ignored != s_replaced); - - // check that replace string contains a replacement character - CHECK(s_replaced.find("\xEF\xBF\xBD") != std::string::npos); - } - - // check that prefix and suffix are preserved - CHECK(s_ignored2.substr(1, 3) == "abc"); - CHECK(s_ignored2.substr(s_ignored2.size() - 4, 3) == "xyz"); - CHECK(s_ignored2_ascii.substr(1, 3) == "abc"); - CHECK(s_ignored2_ascii.substr(s_ignored2_ascii.size() - 4, 3) == "xyz"); - CHECK(s_replaced2.substr(1, 3) == "abc"); - CHECK(s_replaced2.substr(s_replaced2.size() - 4, 3) == "xyz"); - CHECK(s_replaced2_ascii.substr(1, 3) == "abc"); - CHECK(s_replaced2_ascii.substr(s_replaced2_ascii.size() - 4, 3) == "xyz"); -} - -void check_utf8string(bool success_expected, int byte1, int byte2, int byte3, int byte4); - -// create and check a JSON string with up to four UTF-8 bytes -void check_utf8string(bool success_expected, int byte1, int byte2 = -1, int byte3 = -1, int byte4 = -1) -{ - if (++calls % 100000 == 0) - { - std::cout << calls << " of 1246225 UTF-8 strings checked" << std::endl; // NOLINT(performance-avoid-endl) - } - - static std::string json_string; - json_string = "\""; - - CAPTURE(byte1) - json_string += std::string(1, static_cast(byte1)); - - if (byte2 != -1) - { - CAPTURE(byte2) - json_string += std::string(1, static_cast(byte2)); - } - - if (byte3 != -1) - { - CAPTURE(byte3) - json_string += std::string(1, static_cast(byte3)); - } - - if (byte4 != -1) - { - CAPTURE(byte4) - json_string += std::string(1, static_cast(byte4)); - } - - json_string += "\""; - - CAPTURE(json_string) - - json _; - if (success_expected) - { - CHECK_NOTHROW(_ = json::parse(json_string)); - } - else - { - CHECK_THROWS_AS(_ = json::parse(json_string), json::parse_error&); - } -} -} // namespace - -TEST_CASE("Unicode (5/5)" * doctest::skip()) -{ - SECTION("RFC 3629") - { - /* - RFC 3629 describes in Sect. 4 the syntax of UTF-8 byte sequences as - follows: - - A UTF-8 string is a sequence of octets representing a sequence of UCS - characters. An octet sequence is valid UTF-8 only if it matches the - following syntax, which is derived from the rules for encoding UTF-8 - and is expressed in the ABNF of [RFC2234]. - - UTF8-octets = *( UTF8-char ) - UTF8-char = UTF8-1 / UTF8-2 / UTF8-3 / UTF8-4 - UTF8-1 = %x00-7F - UTF8-2 = %xC2-DF UTF8-tail - UTF8-3 = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2( UTF8-tail ) / - %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail ) - UTF8-4 = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) / - %xF4 %x80-8F 2( UTF8-tail ) - UTF8-tail = %x80-BF - */ - - SECTION("UTF8-4 (xF4 x80-8F UTF8-tail UTF8-tail)") - { - SECTION("well-formed") - { - for (int byte1 = 0xF4; byte1 <= 0xF4; ++byte1) - { - for (int byte2 = 0x80; byte2 <= 0x8F; ++byte2) - { - for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3) - { - for (int byte4 = 0x80; byte4 <= 0xBF; ++byte4) - { - check_utf8string(true, byte1, byte2, byte3, byte4); - check_utf8dump(true, byte1, byte2, byte3, byte4); - } - } - } - } - } - - SECTION("ill-formed: missing second byte") - { - for (int byte1 = 0xF4; byte1 <= 0xF4; ++byte1) - { - check_utf8string(false, byte1); - check_utf8dump(false, byte1); - } - } - - SECTION("ill-formed: missing third byte") - { - for (int byte1 = 0xF4; byte1 <= 0xF4; ++byte1) - { - for (int byte2 = 0x80; byte2 <= 0x8F; ++byte2) - { - check_utf8string(false, byte1, byte2); - check_utf8dump(false, byte1, byte2); - } - } - } - - SECTION("ill-formed: missing fourth byte") - { - for (int byte1 = 0xF4; byte1 <= 0xF4; ++byte1) - { - for (int byte2 = 0x80; byte2 <= 0x8F; ++byte2) - { - for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3) - { - check_utf8string(false, byte1, byte2, byte3); - check_utf8dump(false, byte1, byte2, byte3); - } - } - } - } - - SECTION("ill-formed: wrong second byte") - { - for (int byte1 = 0xF4; byte1 <= 0xF4; ++byte1) - { - for (int byte2 = 0x00; byte2 <= 0xFF; ++byte2) - { - // skip correct second byte - if (0x80 <= byte2 && byte2 <= 0x8F) - { - continue; - } - - for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3) - { - for (int byte4 = 0x80; byte4 <= 0xBF; ++byte4) - { - check_utf8string(false, byte1, byte2, byte3, byte4); - check_utf8dump(false, byte1, byte2, byte3, byte4); - } - } - } - } - } - - SECTION("ill-formed: wrong third byte") - { - for (int byte1 = 0xF4; byte1 <= 0xF4; ++byte1) - { - for (int byte2 = 0x80; byte2 <= 0x8F; ++byte2) - { - for (int byte3 = 0x00; byte3 <= 0xFF; ++byte3) - { - // skip correct third byte - if (0x80 <= byte3 && byte3 <= 0xBF) - { - continue; - } - - for (int byte4 = 0x80; byte4 <= 0xBF; ++byte4) - { - check_utf8string(false, byte1, byte2, byte3, byte4); - check_utf8dump(false, byte1, byte2, byte3, byte4); - } - } - } - } - } - - SECTION("ill-formed: wrong fourth byte") - { - for (int byte1 = 0xF4; byte1 <= 0xF4; ++byte1) - { - for (int byte2 = 0x80; byte2 <= 0x8F; ++byte2) - { - for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3) - { - for (int byte4 = 0x00; byte4 <= 0xFF; ++byte4) - { - // skip correct fourth byte - if (0x80 <= byte4 && byte4 <= 0xBF) - { - continue; - } - - check_utf8string(false, byte1, byte2, byte3, byte4); - check_utf8dump(false, byte1, byte2, byte3, byte4); - } - } - } - } - } - } - } -} - -DOCTEST_CLANG_SUPPRESS_WARNING_POP From 21a69230bdaabd05c9b8b1427302f35b11771caf Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Tue, 6 Oct 2026 22:59:34 +0200 Subject: [PATCH 09/12] Create a value before giving it its type (#5585) * Create a value before giving it its type Squashed onto develop from: - Create a value before giving it its type - Skip the failed-allocation test when exceptions are disabled - Keep the created pointer rather than an uninitialized json_value - Skip the vector failed-allocation check for VS 2015 with iterator debugging - Test the remaining to_json overloads with a failing allocation - Skip the to_json allocation-failure section on VS 2015 Debug - Fix false GCC -Warray-bounds error with JSON_DIAGNOSTICS at -O3 (#5744) Signed-off-by: Niels Lohmann * Store the new value with a helper in all to_json constructors Every external_constructor<>::construct now creates the new value first and hands it to basic_json::replace_value(), which destroys the old value, stores the new one before setting its type (as elsewhere in this PR), sets the parents, and checks the invariant. The std::vector and std::valarray overloads use array_t's range constructor, which the other array overloads already rely on. Range views keep their loop, as begin() and end() of a view may have different types. Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann --- .../nlohmann/detail/conversions/to_json.hpp | 139 +++++--------- include/nlohmann/json.hpp | 29 ++- single_include/nlohmann/json.hpp | 168 +++++++---------- tests/CMakeLists.txt | 5 + tests/src/unit-allocator.cpp | 175 ++++++++++++++++++ tests/src/unit-diagnostics-optimized.cpp | 80 ++++++++ 6 files changed, 392 insertions(+), 204 deletions(-) create mode 100644 tests/src/unit-diagnostics-optimized.cpp diff --git a/include/nlohmann/detail/conversions/to_json.hpp b/include/nlohmann/detail/conversions/to_json.hpp index f3b4994f1..8edaee2fe 100644 --- a/include/nlohmann/detail/conversions/to_json.hpp +++ b/include/nlohmann/detail/conversions/to_json.hpp @@ -13,7 +13,6 @@ #include // optional #endif -#include // copy #include // begin, end #include // allocator_traits #include // basic_string, char_traits @@ -39,10 +38,14 @@ namespace detail ////////////////// /* - * Note all external_constructor<>::construct functions need to call - * j.m_data.m_value.destroy(j.m_data.m_type) to avoid a memory leak in case j contains an - * allocated value (e.g., a string). See bug issue + * Note all external_constructor<>::construct functions need to store the new + * value with j.replace_value(), which destroys the old one to avoid a memory + * leak in case j contains an allocated value (e.g., a string). See bug issue * https://github.com/nlohmann/json/issues/2865 for more information. + * + * A value that has to be allocated is created before the old one is destroyed: + * were it the other way around, an exception while creating the new value would + * leave j with the type of the new value, but the pointer to the destroyed old one. */ template struct external_constructor; @@ -53,10 +56,7 @@ struct external_constructor template static void construct(BasicJsonType& j, typename BasicJsonType::boolean_t b) noexcept { - j.m_data.m_value.destroy(j.m_data.m_type); - j.m_data.m_type = value_t::boolean; - j.m_data.m_value = b; - j.assert_invariant(); + j.replace_value(value_t::boolean, b); } }; @@ -66,19 +66,15 @@ struct external_constructor template static void construct(BasicJsonType& j, const typename BasicJsonType::string_t& s) { - j.m_data.m_value.destroy(j.m_data.m_type); - j.m_data.m_type = value_t::string; - j.m_data.m_value = s; - j.assert_invariant(); + const typename BasicJsonType::json_value value(s); + j.replace_value(value_t::string, value); } template static void construct(BasicJsonType& j, typename BasicJsonType::string_t&& s) { - j.m_data.m_value.destroy(j.m_data.m_type); - j.m_data.m_type = value_t::string; - j.m_data.m_value = std::move(s); - j.assert_invariant(); + const typename BasicJsonType::json_value value(std::move(s)); + j.replace_value(value_t::string, value); } template < typename BasicJsonType, typename CompatibleStringType, @@ -86,10 +82,8 @@ struct external_constructor int > = 0 > static void construct(BasicJsonType& j, const CompatibleStringType& str) { - j.m_data.m_value.destroy(j.m_data.m_type); - j.m_data.m_type = value_t::string; - j.m_data.m_value.string = j.template create(str); - j.assert_invariant(); + const typename BasicJsonType::json_value value(j.template create(str)); + j.replace_value(value_t::string, value); } }; @@ -99,19 +93,15 @@ struct external_constructor template static void construct(BasicJsonType& j, const typename BasicJsonType::binary_t& b) { - j.m_data.m_value.destroy(j.m_data.m_type); - j.m_data.m_type = value_t::binary; - j.m_data.m_value = typename BasicJsonType::binary_t(b); - j.assert_invariant(); + const typename BasicJsonType::json_value value(b); + j.replace_value(value_t::binary, value); } template static void construct(BasicJsonType& j, typename BasicJsonType::binary_t&& b) { - j.m_data.m_value.destroy(j.m_data.m_type); - j.m_data.m_type = value_t::binary; - j.m_data.m_value = typename BasicJsonType::binary_t(std::move(b)); - j.assert_invariant(); + const typename BasicJsonType::json_value value(std::move(b)); + j.replace_value(value_t::binary, value); } }; @@ -121,10 +111,7 @@ struct external_constructor template static void construct(BasicJsonType& j, typename BasicJsonType::number_float_t val) noexcept { - j.m_data.m_value.destroy(j.m_data.m_type); - j.m_data.m_type = value_t::number_float; - j.m_data.m_value = val; - j.assert_invariant(); + j.replace_value(value_t::number_float, val); } }; @@ -134,10 +121,7 @@ struct external_constructor template static void construct(BasicJsonType& j, typename BasicJsonType::number_unsigned_t val) noexcept { - j.m_data.m_value.destroy(j.m_data.m_type); - j.m_data.m_type = value_t::number_unsigned; - j.m_data.m_value = val; - j.assert_invariant(); + j.replace_value(value_t::number_unsigned, val); } }; @@ -147,10 +131,7 @@ struct external_constructor template static void construct(BasicJsonType& j, typename BasicJsonType::number_integer_t val) noexcept { - j.m_data.m_value.destroy(j.m_data.m_type); - j.m_data.m_type = value_t::number_integer; - j.m_data.m_value = val; - j.assert_invariant(); + j.replace_value(value_t::number_integer, val); } }; @@ -160,21 +141,15 @@ struct external_constructor template static void construct(BasicJsonType& j, const typename BasicJsonType::array_t& arr) { - j.m_data.m_value.destroy(j.m_data.m_type); - j.m_data.m_type = value_t::array; - j.m_data.m_value = arr; - j.set_parents(); - j.assert_invariant(); + const typename BasicJsonType::json_value value(arr); + j.replace_value(value_t::array, value); } template static void construct(BasicJsonType& j, typename BasicJsonType::array_t&& arr) { - j.m_data.m_value.destroy(j.m_data.m_type); - j.m_data.m_type = value_t::array; - j.m_data.m_value = std::move(arr); - j.set_parents(); - j.assert_invariant(); + const typename BasicJsonType::json_value value(std::move(arr)); + j.replace_value(value_t::array, value); } template < typename BasicJsonType, typename CompatibleArrayType, @@ -188,39 +163,23 @@ struct external_constructor using std::begin; using std::end; - j.m_data.m_value.destroy(j.m_data.m_type); - j.m_data.m_type = value_t::array; - j.m_data.m_value.array = j.template create(begin(arr), end(arr)); - j.set_parents(); - j.assert_invariant(); + const typename BasicJsonType::json_value value(j.template create(begin(arr), end(arr))); + j.replace_value(value_t::array, value); } template static void construct(BasicJsonType& j, const std::vector& arr) { - j.m_data.m_value.destroy(j.m_data.m_type); - j.m_data.m_type = value_t::array; - j.m_data.m_value = value_t::array; - j.m_data.m_value.array->reserve(arr.size()); - for (const bool x : arr) - { - j.m_data.m_value.array->push_back(x); - j.set_parent(j.m_data.m_value.array->back()); - } - j.assert_invariant(); + const typename BasicJsonType::json_value value(j.template create(arr.begin(), arr.end())); + j.replace_value(value_t::array, value); } template::value, int> = 0> static void construct(BasicJsonType& j, const std::valarray& arr) { - j.m_data.m_value.destroy(j.m_data.m_type); - j.m_data.m_type = value_t::array; - j.m_data.m_value = value_t::array; - j.m_data.m_value.array->resize(arr.size()); - std::copy(std::begin(arr), std::end(arr), j.m_data.m_value.array->begin()); - j.set_parents(); - j.assert_invariant(); + const typename BasicJsonType::json_value value(j.template create(std::begin(arr), std::end(arr))); + j.replace_value(value_t::array, value); } #if JSON_HAS_RANGE_VIEW_CONVERSION @@ -228,18 +187,15 @@ struct external_constructor enable_if_t>::value, int> = 0> static void construct(BasicJsonType& j, CompatibleArrayType && arr) { - j.m_data.m_value.destroy(j.m_data.m_type); - j.m_data.m_type = value_t::array; - j.m_data.m_value = value_t::array; + // no range constructor: a view's begin() and end() may have different + // types, and the view may only be iterable once + typename BasicJsonType::array_t elements; for (auto&& x : std::forward(arr)) { - j.m_data.m_value.array->push_back(x); + elements.push_back(x); } - // set the parents only once all elements are in place: a push_back - // that reallocates moves the earlier elements, which does not keep - // their parent pointers - j.set_parents(); - j.assert_invariant(); + const typename BasicJsonType::json_value value(std::move(elements)); + j.replace_value(value_t::array, value); } #endif }; @@ -250,21 +206,15 @@ struct external_constructor template static void construct(BasicJsonType& j, const typename BasicJsonType::object_t& obj) { - j.m_data.m_value.destroy(j.m_data.m_type); - j.m_data.m_type = value_t::object; - j.m_data.m_value = obj; - j.set_parents(); - j.assert_invariant(); + const typename BasicJsonType::json_value value(obj); + j.replace_value(value_t::object, value); } template static void construct(BasicJsonType& j, typename BasicJsonType::object_t&& obj) { - j.m_data.m_value.destroy(j.m_data.m_type); - j.m_data.m_type = value_t::object; - j.m_data.m_value = std::move(obj); - j.set_parents(); - j.assert_invariant(); + const typename BasicJsonType::json_value value(std::move(obj)); + j.replace_value(value_t::object, value); } template < typename BasicJsonType, typename CompatibleObjectType, @@ -274,11 +224,8 @@ struct external_constructor using std::begin; using std::end; - j.m_data.m_value.destroy(j.m_data.m_type); - j.m_data.m_type = value_t::object; - j.m_data.m_value.object = j.template create(begin(obj), end(obj)); - j.set_parents(); - j.assert_invariant(); + const typename BasicJsonType::json_value value(j.template create(begin(obj), end(obj))); + j.replace_value(value_t::object, value); } }; diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 4eeb326b8..788cc999c 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -676,6 +676,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// constructor for rvalue binary arrays (internal type) json_value(binary_t&& value) : binary(create(std::move(value))) {} + /// constructors taking ownership of an already created value + explicit json_value(string_t* value) noexcept : string(value) {} + explicit json_value(object_t* value) noexcept : object(value) {} + explicit json_value(array_t* value) noexcept : array(value) {} + private: // raw, allocation-free transfer of m_data from src to dst: no // set_parents()/assert_invariant() (the former is O(#children) per @@ -1006,6 +1011,18 @@ public: #endif } + /// @brief replace the stored value with an already created one + /// The new value must be created before calling this function: if its + /// creation throws, the current value is left untouched. + void replace_value(value_t t, const json_value& v) noexcept + { + m_data.m_value.destroy(m_data.m_type); + m_data.m_value = v; + m_data.m_type = t; + set_parents(); + assert_invariant(); + } + iterator set_parents(iterator it, std::ptrdiff_t count_set_parents) { #if JSON_DIAGNOSTICS @@ -2265,8 +2282,8 @@ public: if (is_an_object) { // the initializer list is a list of pairs -> create an object - m_data.m_type = value_t::object; m_data.m_value = value_t::object; + m_data.m_type = value_t::object; for (auto& element_ref : init) { @@ -2288,8 +2305,8 @@ public: } #endif // the initializer list describes an array -> create an array - m_data.m_type = value_t::array; m_data.m_value.array = create(init.begin(), init.end()); + m_data.m_type = value_t::array; } set_parents(); @@ -2302,8 +2319,8 @@ public: static basic_json binary(const typename binary_t::container_type& init) { auto res = basic_json(); - res.m_data.m_type = value_t::binary; res.m_data.m_value = init; + res.m_data.m_type = value_t::binary; return res; } @@ -2313,8 +2330,8 @@ public: static basic_json binary(const typename binary_t::container_type& init, typename binary_t::subtype_type subtype) { auto res = basic_json(); - res.m_data.m_type = value_t::binary; res.m_data.m_value = binary_t(init, subtype); + res.m_data.m_type = value_t::binary; return res; } @@ -2324,8 +2341,8 @@ public: static basic_json binary(typename binary_t::container_type&& init) { auto res = basic_json(); - res.m_data.m_type = value_t::binary; res.m_data.m_value = std::move(init); + res.m_data.m_type = value_t::binary; return res; } @@ -2335,8 +2352,8 @@ public: static basic_json binary(typename binary_t::container_type&& init, typename binary_t::subtype_type subtype) { auto res = basic_json(); - res.m_data.m_type = value_t::binary; res.m_data.m_value = binary_t(std::move(init), subtype); + res.m_data.m_type = value_t::binary; return res; } diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index be32959d5..7d19b53af 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -6193,7 +6193,6 @@ NLOHMANN_JSON_NAMESPACE_END #include // optional #endif -#include // copy #include // begin, end #include // allocator_traits #include // basic_string, char_traits @@ -6833,10 +6832,14 @@ namespace detail ////////////////// /* - * Note all external_constructor<>::construct functions need to call - * j.m_data.m_value.destroy(j.m_data.m_type) to avoid a memory leak in case j contains an - * allocated value (e.g., a string). See bug issue + * Note all external_constructor<>::construct functions need to store the new + * value with j.replace_value(), which destroys the old one to avoid a memory + * leak in case j contains an allocated value (e.g., a string). See bug issue * https://github.com/nlohmann/json/issues/2865 for more information. + * + * A value that has to be allocated is created before the old one is destroyed: + * were it the other way around, an exception while creating the new value would + * leave j with the type of the new value, but the pointer to the destroyed old one. */ template struct external_constructor; @@ -6847,10 +6850,7 @@ struct external_constructor template static void construct(BasicJsonType& j, typename BasicJsonType::boolean_t b) noexcept { - j.m_data.m_value.destroy(j.m_data.m_type); - j.m_data.m_type = value_t::boolean; - j.m_data.m_value = b; - j.assert_invariant(); + j.replace_value(value_t::boolean, b); } }; @@ -6860,19 +6860,15 @@ struct external_constructor template static void construct(BasicJsonType& j, const typename BasicJsonType::string_t& s) { - j.m_data.m_value.destroy(j.m_data.m_type); - j.m_data.m_type = value_t::string; - j.m_data.m_value = s; - j.assert_invariant(); + const typename BasicJsonType::json_value value(s); + j.replace_value(value_t::string, value); } template static void construct(BasicJsonType& j, typename BasicJsonType::string_t&& s) { - j.m_data.m_value.destroy(j.m_data.m_type); - j.m_data.m_type = value_t::string; - j.m_data.m_value = std::move(s); - j.assert_invariant(); + const typename BasicJsonType::json_value value(std::move(s)); + j.replace_value(value_t::string, value); } template < typename BasicJsonType, typename CompatibleStringType, @@ -6880,10 +6876,8 @@ struct external_constructor int > = 0 > static void construct(BasicJsonType& j, const CompatibleStringType& str) { - j.m_data.m_value.destroy(j.m_data.m_type); - j.m_data.m_type = value_t::string; - j.m_data.m_value.string = j.template create(str); - j.assert_invariant(); + const typename BasicJsonType::json_value value(j.template create(str)); + j.replace_value(value_t::string, value); } }; @@ -6893,19 +6887,15 @@ struct external_constructor template static void construct(BasicJsonType& j, const typename BasicJsonType::binary_t& b) { - j.m_data.m_value.destroy(j.m_data.m_type); - j.m_data.m_type = value_t::binary; - j.m_data.m_value = typename BasicJsonType::binary_t(b); - j.assert_invariant(); + const typename BasicJsonType::json_value value(b); + j.replace_value(value_t::binary, value); } template static void construct(BasicJsonType& j, typename BasicJsonType::binary_t&& b) { - j.m_data.m_value.destroy(j.m_data.m_type); - j.m_data.m_type = value_t::binary; - j.m_data.m_value = typename BasicJsonType::binary_t(std::move(b)); - j.assert_invariant(); + const typename BasicJsonType::json_value value(std::move(b)); + j.replace_value(value_t::binary, value); } }; @@ -6915,10 +6905,7 @@ struct external_constructor template static void construct(BasicJsonType& j, typename BasicJsonType::number_float_t val) noexcept { - j.m_data.m_value.destroy(j.m_data.m_type); - j.m_data.m_type = value_t::number_float; - j.m_data.m_value = val; - j.assert_invariant(); + j.replace_value(value_t::number_float, val); } }; @@ -6928,10 +6915,7 @@ struct external_constructor template static void construct(BasicJsonType& j, typename BasicJsonType::number_unsigned_t val) noexcept { - j.m_data.m_value.destroy(j.m_data.m_type); - j.m_data.m_type = value_t::number_unsigned; - j.m_data.m_value = val; - j.assert_invariant(); + j.replace_value(value_t::number_unsigned, val); } }; @@ -6941,10 +6925,7 @@ struct external_constructor template static void construct(BasicJsonType& j, typename BasicJsonType::number_integer_t val) noexcept { - j.m_data.m_value.destroy(j.m_data.m_type); - j.m_data.m_type = value_t::number_integer; - j.m_data.m_value = val; - j.assert_invariant(); + j.replace_value(value_t::number_integer, val); } }; @@ -6954,21 +6935,15 @@ struct external_constructor template static void construct(BasicJsonType& j, const typename BasicJsonType::array_t& arr) { - j.m_data.m_value.destroy(j.m_data.m_type); - j.m_data.m_type = value_t::array; - j.m_data.m_value = arr; - j.set_parents(); - j.assert_invariant(); + const typename BasicJsonType::json_value value(arr); + j.replace_value(value_t::array, value); } template static void construct(BasicJsonType& j, typename BasicJsonType::array_t&& arr) { - j.m_data.m_value.destroy(j.m_data.m_type); - j.m_data.m_type = value_t::array; - j.m_data.m_value = std::move(arr); - j.set_parents(); - j.assert_invariant(); + const typename BasicJsonType::json_value value(std::move(arr)); + j.replace_value(value_t::array, value); } template < typename BasicJsonType, typename CompatibleArrayType, @@ -6982,39 +6957,23 @@ struct external_constructor using std::begin; using std::end; - j.m_data.m_value.destroy(j.m_data.m_type); - j.m_data.m_type = value_t::array; - j.m_data.m_value.array = j.template create(begin(arr), end(arr)); - j.set_parents(); - j.assert_invariant(); + const typename BasicJsonType::json_value value(j.template create(begin(arr), end(arr))); + j.replace_value(value_t::array, value); } template static void construct(BasicJsonType& j, const std::vector& arr) { - j.m_data.m_value.destroy(j.m_data.m_type); - j.m_data.m_type = value_t::array; - j.m_data.m_value = value_t::array; - j.m_data.m_value.array->reserve(arr.size()); - for (const bool x : arr) - { - j.m_data.m_value.array->push_back(x); - j.set_parent(j.m_data.m_value.array->back()); - } - j.assert_invariant(); + const typename BasicJsonType::json_value value(j.template create(arr.begin(), arr.end())); + j.replace_value(value_t::array, value); } template::value, int> = 0> static void construct(BasicJsonType& j, const std::valarray& arr) { - j.m_data.m_value.destroy(j.m_data.m_type); - j.m_data.m_type = value_t::array; - j.m_data.m_value = value_t::array; - j.m_data.m_value.array->resize(arr.size()); - std::copy(std::begin(arr), std::end(arr), j.m_data.m_value.array->begin()); - j.set_parents(); - j.assert_invariant(); + const typename BasicJsonType::json_value value(j.template create(std::begin(arr), std::end(arr))); + j.replace_value(value_t::array, value); } #if JSON_HAS_RANGE_VIEW_CONVERSION @@ -7022,18 +6981,15 @@ struct external_constructor enable_if_t>::value, int> = 0> static void construct(BasicJsonType& j, CompatibleArrayType && arr) { - j.m_data.m_value.destroy(j.m_data.m_type); - j.m_data.m_type = value_t::array; - j.m_data.m_value = value_t::array; + // no range constructor: a view's begin() and end() may have different + // types, and the view may only be iterable once + typename BasicJsonType::array_t elements; for (auto&& x : std::forward(arr)) { - j.m_data.m_value.array->push_back(x); + elements.push_back(x); } - // set the parents only once all elements are in place: a push_back - // that reallocates moves the earlier elements, which does not keep - // their parent pointers - j.set_parents(); - j.assert_invariant(); + const typename BasicJsonType::json_value value(std::move(elements)); + j.replace_value(value_t::array, value); } #endif }; @@ -7044,21 +7000,15 @@ struct external_constructor template static void construct(BasicJsonType& j, const typename BasicJsonType::object_t& obj) { - j.m_data.m_value.destroy(j.m_data.m_type); - j.m_data.m_type = value_t::object; - j.m_data.m_value = obj; - j.set_parents(); - j.assert_invariant(); + const typename BasicJsonType::json_value value(obj); + j.replace_value(value_t::object, value); } template static void construct(BasicJsonType& j, typename BasicJsonType::object_t&& obj) { - j.m_data.m_value.destroy(j.m_data.m_type); - j.m_data.m_type = value_t::object; - j.m_data.m_value = std::move(obj); - j.set_parents(); - j.assert_invariant(); + const typename BasicJsonType::json_value value(std::move(obj)); + j.replace_value(value_t::object, value); } template < typename BasicJsonType, typename CompatibleObjectType, @@ -7068,11 +7018,8 @@ struct external_constructor using std::begin; using std::end; - j.m_data.m_value.destroy(j.m_data.m_type); - j.m_data.m_type = value_t::object; - j.m_data.m_value.object = j.template create(begin(obj), end(obj)); - j.set_parents(); - j.assert_invariant(); + const typename BasicJsonType::json_value value(j.template create(begin(obj), end(obj))); + j.replace_value(value_t::object, value); } }; @@ -27921,6 +27868,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// constructor for rvalue binary arrays (internal type) json_value(binary_t&& value) : binary(create(std::move(value))) {} + /// constructors taking ownership of an already created value + explicit json_value(string_t* value) noexcept : string(value) {} + explicit json_value(object_t* value) noexcept : object(value) {} + explicit json_value(array_t* value) noexcept : array(value) {} + private: // raw, allocation-free transfer of m_data from src to dst: no // set_parents()/assert_invariant() (the former is O(#children) per @@ -28251,6 +28203,18 @@ public: #endif } + /// @brief replace the stored value with an already created one + /// The new value must be created before calling this function: if its + /// creation throws, the current value is left untouched. + void replace_value(value_t t, const json_value& v) noexcept + { + m_data.m_value.destroy(m_data.m_type); + m_data.m_value = v; + m_data.m_type = t; + set_parents(); + assert_invariant(); + } + iterator set_parents(iterator it, std::ptrdiff_t count_set_parents) { #if JSON_DIAGNOSTICS @@ -29510,8 +29474,8 @@ public: if (is_an_object) { // the initializer list is a list of pairs -> create an object - m_data.m_type = value_t::object; m_data.m_value = value_t::object; + m_data.m_type = value_t::object; for (auto& element_ref : init) { @@ -29533,8 +29497,8 @@ public: } #endif // the initializer list describes an array -> create an array - m_data.m_type = value_t::array; m_data.m_value.array = create(init.begin(), init.end()); + m_data.m_type = value_t::array; } set_parents(); @@ -29547,8 +29511,8 @@ public: static basic_json binary(const typename binary_t::container_type& init) { auto res = basic_json(); - res.m_data.m_type = value_t::binary; res.m_data.m_value = init; + res.m_data.m_type = value_t::binary; return res; } @@ -29558,8 +29522,8 @@ public: static basic_json binary(const typename binary_t::container_type& init, typename binary_t::subtype_type subtype) { auto res = basic_json(); - res.m_data.m_type = value_t::binary; res.m_data.m_value = binary_t(init, subtype); + res.m_data.m_type = value_t::binary; return res; } @@ -29569,8 +29533,8 @@ public: static basic_json binary(typename binary_t::container_type&& init) { auto res = basic_json(); - res.m_data.m_type = value_t::binary; res.m_data.m_value = std::move(init); + res.m_data.m_type = value_t::binary; return res; } @@ -29580,8 +29544,8 @@ public: static basic_json binary(typename binary_t::container_type&& init, typename binary_t::subtype_type subtype) { auto res = basic_json(); - res.m_data.m_type = value_t::binary; res.m_data.m_value = binary_t(std::move(init), subtype); + res.m_data.m_type = value_t::binary; return res; } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index f98ab7593..de415897f 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -138,6 +138,11 @@ json_test_set_test_options(test-disabled_exceptions # only the #972 regression test needs thirdparty/fifo_map on its include path json_test_set_test_options(test-regression1 LINK_LIBRARIES fifo_map_include) +# GCC's false -Warray-bounds error with JSON_DIAGNOSTICS only shows up when optimizing (#5742) +json_test_set_test_options(test-diagnostics-optimized + COMPILE_OPTIONS $<$:-O3 -Werror=array-bounds> +) + ############################################################################# # add unit tests ############################################################################# diff --git a/tests/src/unit-allocator.cpp b/tests/src/unit-allocator.cpp index d2e23c1a7..07eb21c4e 100644 --- a/tests/src/unit-allocator.cpp +++ b/tests/src/unit-allocator.cpp @@ -12,6 +12,11 @@ #include using nlohmann::json; +#include +#if JSON_HAS_RANGES + #include +#endif + namespace { // special test case to check if memory is leaked if constructor throws @@ -671,3 +676,173 @@ TEST_CASE("destructor performs no allocation, only deallocation") CHECK(counting_allocator_deallocations > deallocations_before); } } + +// the no-exceptions CI job skips every CHECK_THROWS_AS, which would leave +// next_construct_fails set for the next allocation outside a check +#if !defined(JSON_NOEXCEPTION) +TEST_CASE("a failed allocation leaves the value unchanged") +{ + // create JSON type using the throwing allocator + using my_json = nlohmann::basic_json; + + // Each of these creates a string, array, object, or binary value. The + // value must be created before the type is changed: otherwise, a failed + // creation left a value of the new type without anything behind it (an + // assertion in its destructor, a null pointer everywhere else) or, when + // an old value was destroyed first, with a pointer to that destroyed one. + + SECTION("creating a binary value") + { + const std::vector bytes = {1, 2, 3}; + my_json _; + + next_construct_fails = true; + CHECK_THROWS_AS(_ = my_json::binary(bytes), std::bad_alloc&); + next_construct_fails = true; + CHECK_THROWS_AS(_ = my_json::binary(bytes, 42), std::bad_alloc&); + next_construct_fails = true; + CHECK_THROWS_AS(_ = my_json::binary(std::vector(bytes)), std::bad_alloc&); + next_construct_fails = true; + CHECK_THROWS_AS(_ = my_json::binary(std::vector(bytes), 42), std::bad_alloc&); + next_construct_fails = false; + } + + SECTION("turning a null value into an array or object") + { + my_json j; + + next_construct_fails = true; + CHECK_THROWS_AS(j[0], std::bad_alloc&); + CHECK(j.is_null()); + + next_construct_fails = true; + CHECK_THROWS_AS(j["key"], std::bad_alloc&); + CHECK(j.is_null()); + +#ifdef JSON_HAS_CPP_17 + next_construct_fails = true; + CHECK_THROWS_AS(j[std::string_view("key")], std::bad_alloc&); + CHECK(j.is_null()); +#endif + + next_construct_fails = true; + CHECK_THROWS_AS(j.push_back(my_json(1)), std::bad_alloc&); + CHECK(j.is_null()); + + const my_json one = 1; + next_construct_fails = true; + CHECK_THROWS_AS(j.push_back(one), std::bad_alloc&); + CHECK(j.is_null()); + + next_construct_fails = true; + CHECK_THROWS_AS(j.push_back(my_json::object_t::value_type("key", 1)), std::bad_alloc&); + CHECK(j.is_null()); + + next_construct_fails = true; + CHECK_THROWS_AS(j.emplace_back(1), std::bad_alloc&); + CHECK(j.is_null()); + + next_construct_fails = true; + CHECK_THROWS_AS(j.emplace("key", 1), std::bad_alloc&); + CHECK(j.is_null()); + + const my_json object = {{"key", 1}}; + next_construct_fails = true; + CHECK_THROWS_AS(j.update(object), std::bad_alloc&); + CHECK(j.is_null()); + + next_construct_fails = false; + } + + // With iterator debugging, VS 2015's containers construct a proxy with the + // allocator in constructors that cannot report its failure, so a failing + // allocator crashes this section there (SIGSEGV with VS 2015 Debug x86). +#if !(defined(_MSC_VER) && _MSC_VER < 1910 && defined(_ITERATOR_DEBUG_LEVEL) && _ITERATOR_DEBUG_LEVEL > 0) + SECTION("converting into an existing value") + { + // to_json replaces the value it is given; the old one must survive a + // failed creation of the new one + my_json j = "old"; + + next_construct_fails = true; + CHECK_THROWS_AS(nlohmann::to_json(j, std::string("new")), std::bad_alloc&); + CHECK(j == "old"); + + next_construct_fails = true; + CHECK_THROWS_AS(nlohmann::to_json(j, std::vector {1, 2}), std::bad_alloc&); + CHECK(j == "old"); + + next_construct_fails = true; + CHECK_THROWS_AS(nlohmann::to_json(j, std::vector {true, false}), std::bad_alloc&); + CHECK(j == "old"); + + next_construct_fails = true; + CHECK_THROWS_AS(nlohmann::to_json(j, std::map {{"a", 1}}), std::bad_alloc&); + CHECK(j == "old"); + + next_construct_fails = true; + CHECK_THROWS_AS(nlohmann::to_json(j, my_json::binary_t({1, 2})), std::bad_alloc&); + CHECK(j == "old"); + + // the overloads for lvalues of the value types, for the value types + // themselves, and for the remaining compatible types + const std::string string = "new"; + next_construct_fails = true; + CHECK_THROWS_AS(nlohmann::to_json(j, string), std::bad_alloc&); + CHECK(j == "old"); + + next_construct_fails = true; + CHECK_THROWS_AS(nlohmann::to_json(j, "new"), std::bad_alloc&); + CHECK(j == "old"); + + // to_json only moves a binary value that it converted from another + // container type, which my_json's std::vector is not + using binary_constructor = nlohmann::detail::external_constructor; + next_construct_fails = true; + CHECK_THROWS_AS(binary_constructor::construct(j, my_json::binary_t({1, 2})), std::bad_alloc&); + CHECK(j == "old"); + + my_json::array_t array = {1, 2}; + next_construct_fails = true; + CHECK_THROWS_AS(nlohmann::to_json(j, array), std::bad_alloc&); + CHECK(j == "old"); + next_construct_fails = true; + CHECK_THROWS_AS(nlohmann::to_json(j, std::move(array)), std::bad_alloc&); + CHECK(j == "old"); + + my_json::object_t object = {{"a", 1}}; + next_construct_fails = true; + CHECK_THROWS_AS(nlohmann::to_json(j, object), std::bad_alloc&); + CHECK(j == "old"); + next_construct_fails = true; + CHECK_THROWS_AS(nlohmann::to_json(j, std::move(object)), std::bad_alloc&); + CHECK(j == "old"); + + next_construct_fails = true; + CHECK_THROWS_AS(nlohmann::to_json(j, std::valarray {1, 2}), std::bad_alloc&); + CHECK(j == "old"); + +#if JSON_HAS_RANGES && !defined(__MINGW32__) + const std::vector numbers = {1, 2}; + next_construct_fails = true; + CHECK_THROWS_AS(nlohmann::to_json(j, numbers | std::views::filter([](int /*unused*/) + { + return true; + })), std::bad_alloc&); + CHECK(j == "old"); +#endif + + next_construct_fails = false; + nlohmann::to_json(j, std::vector {1, 2}); + CHECK(j == my_json({1, 2})); + } +#endif +} +#endif diff --git a/tests/src/unit-diagnostics-optimized.cpp b/tests/src/unit-diagnostics-optimized.cpp new file mode 100644 index 000000000..a020c2fe9 --- /dev/null +++ b/tests/src/unit-diagnostics-optimized.cpp @@ -0,0 +1,80 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +// Regression test for https://github.com/nlohmann/json/issues/5742: with +// JSON_DIAGNOSTICS, GCC (12 to at least 16) reported a false -Warray-bounds +// error in the inlined set_parents() at -O3. The type of a new string was set +// before the string was allocated, so GCC had to assume that operator new +// could change it again and checked the object branch of set_parents() +// against the string's allocation. Setting the type after creating the value +// avoids this. The warning depends on GCC's inlining decisions, so the +// sections cover two patterns that trigger it on different GCC versions +// (#4819 and #5742). +// On GCC, this file is compiled with -O3 -Werror=array-bounds (see +// tests/CMakeLists.txt), so the test fails to build if the warning returns. + +#include "doctest_compatibility.h" + +#ifdef JSON_DIAGNOSTICS + #undef JSON_DIAGNOSTICS +#endif + +#define JSON_DIAGNOSTICS 1 + +#include +using nlohmann::json; + +#include +#include +#include +#include + +namespace +{ +enum class diag_color +{ + red, + green, + blue +}; + +void to_json(json& j, const diag_color& c) +{ + static const std::pair m[] = // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + { + {diag_color::red, "r"}, + {diag_color::green, "g"}, + {diag_color::blue, "b"}, + }; + const auto* it = std::find_if(std::begin(m), std::end(m), [c](const std::pair& p) + { + return p.first == c; + }); + j = it->second; +} +} // namespace + +TEST_CASE("diagnostics with optimization") +{ + SECTION("issue #4819 - object in vector") + { + std::vector jsons{}; + jsons.emplace_back(json({{"key", "value"}})); + CHECK(jsons.back()["key"] == "value"); + } + + SECTION("issue #5742 - string values from a static table") + { + json j = json::array(); + j.push_back(diag_color::red); + j.push_back(diag_color::green); + j.push_back(diag_color::blue); + CHECK(j.dump() == R"(["r","g","b"])"); + CHECK_THROWS_WITH_AS(j[1].get(), "[json.exception.type_error.302] (/1) type must be number, but is string", json::type_error); + } +} From 675e519966404f2e2ea0160c5ea28d4a3607fc2a Mon Sep 17 00:00:00 2001 From: Mohd Quamar Tyagi <104281681+Tyagiquamar@users.noreply.github.com> Date: Wed, 7 Oct 2026 11:08:24 +0530 Subject: [PATCH 10/12] Allow SAX parsing of tagged CBOR (#5740) * Allow SAX parsing of tagged CBOR Signed-off-by: Tyagiquamar * Consolidate sax_parse overloads with default tag_handler parameter Signed-off-by: Tyagiquamar * Add version history entry for tag_handler in sax_parse documentation --------- Signed-off-by: Tyagiquamar Signed-off-by: Niels Lohmann Co-authored-by: Niels Lohmann --- .../docs/api/basic_json/cbor_tag_handler_t.md | 4 +- docs/mkdocs/docs/api/basic_json/sax_parse.md | 13 ++++-- include/nlohmann/json.hpp | 15 ++++--- tests/src/unit-regression3.cpp | 40 +++++++++++++++++++ 4 files changed, 61 insertions(+), 11 deletions(-) diff --git a/docs/mkdocs/docs/api/basic_json/cbor_tag_handler_t.md b/docs/mkdocs/docs/api/basic_json/cbor_tag_handler_t.md index dfeb5d693..55a7954d5 100644 --- a/docs/mkdocs/docs/api/basic_json/cbor_tag_handler_t.md +++ b/docs/mkdocs/docs/api/basic_json/cbor_tag_handler_t.md @@ -9,10 +9,10 @@ enum class cbor_tag_handler_t }; ``` -This enumeration is used in the [`from_cbor`](from_cbor.md) function to choose how to treat tags: +This enumeration is used in [`from_cbor`](from_cbor.md) and [`sax_parse`](sax_parse.md) to choose how to treat tags: error -: throw a `parse_error` exception in case of a tag +: report a parse error in case of a tag (the `from_cbor` overloads throw a `parse_error` exception by default) ignore : ignore tags diff --git a/docs/mkdocs/docs/api/basic_json/sax_parse.md b/docs/mkdocs/docs/api/basic_json/sax_parse.md index 0d4b8da8a..fc48c9e12 100644 --- a/docs/mkdocs/docs/api/basic_json/sax_parse.md +++ b/docs/mkdocs/docs/api/basic_json/sax_parse.md @@ -8,7 +8,8 @@ static bool sax_parse(InputType&& i, input_format_t format = input_format_t::json, const bool strict = true, const bool ignore_comments = false, - const bool ignore_trailing_commas = false); + const bool ignore_trailing_commas = false, + const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error); // (2) template @@ -17,13 +18,14 @@ static bool sax_parse(IteratorType first, SentinelType last, input_format_t format = input_format_t::json, const bool strict = true, const bool ignore_comments = false, - const bool ignore_trailing_commas = false); + const bool ignore_trailing_commas = false, + const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error); ``` Read from input and generate SAX events 1. Read from a compatible input. -2. Read from a pair of character iterators, or an iterator and a sentinel of a different type (C++20 ranges support) +2. Read from a pair of character iterators, or an iterator and a sentinel of a different type (C++20 ranges support). The value_type of the iterator must be an integral type with a size of 1, 2, or 4 bytes, which will be interpreted respectively as UTF-8, UTF-16, and UTF-32. If `SentinelType` differs from `IteratorType`, it must be comparable to @@ -82,6 +84,10 @@ The SAX event lister must follow the interface of [`json_sax`](../json_sax/index : whether trailing commas in arrays or objects should be ignored and treated like whitespace (`#!cpp true`) or yield a parse error (`#!cpp false`); (optional, `#!cpp false` by default) +`tag_handler` (in) +: how to handle CBOR tags; see [`cbor_tag_handler_t`](cbor_tag_handler_t.md). Ignored for formats other than CBOR + (optional, `cbor_tag_handler_t::error` by default). + `first` (in) : iterator to the start of a character range @@ -137,6 +143,7 @@ A UTF-8 byte order mark is silently ignored. - Added in version 3.2.0. - Ignoring comments via `ignore_comments` added in version 3.9.0. - Added `ignore_trailing_commas` in version 3.13.0. +- Added `tag_handler` in version 3.13.0. - Extended container support (1) to include types with lvalue-only ADL `begin`/`end` (matching `std::begin`/`std::end` semantics) in version 3.13.0. - Extended overload (2) to accept heterogeneous iterator+sentinel pairs (C++20 ranges support) in version 3.13.0. - `JSON_PRECISE_STREAM_POSITION` added in version 3.13.0 to optionally leave a `#!cpp std::istream` positioned right diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 788cc999c..397b253ad 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -5671,12 +5671,13 @@ public: input_format_t format = input_format_t::json, const bool strict = true, const bool ignore_comments = false, - const bool ignore_trailing_commas = false) + const bool ignore_trailing_commas = false, + const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error) { auto ia = detail::input_adapter(std::forward(i)); return format == input_format_t::json ? parser(std::move(ia), nullptr, true, ignore_comments, ignore_trailing_commas).sax_parse(sax, strict) - : detail::binary_reader(std::move(ia), format).sax_parse(sax, strict); + : detail::binary_reader(std::move(ia), format).sax_parse(sax, strict, tag_handler); } /// @brief generate SAX events (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -5688,12 +5689,13 @@ public: input_format_t format = input_format_t::json, const bool strict = true, const bool ignore_comments = false, - const bool ignore_trailing_commas = false) + const bool ignore_trailing_commas = false, + const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error) { auto ia = detail::input_adapter(std::move(first), std::move(last)); return format == input_format_t::json ? parser(std::move(ia), nullptr, true, ignore_comments, ignore_trailing_commas).sax_parse(sax, strict) - : detail::binary_reader(std::move(ia), format).sax_parse(sax, strict); + : detail::binary_reader(std::move(ia), format).sax_parse(sax, strict, tag_handler); } /// @brief generate SAX events @@ -5721,7 +5723,8 @@ public: input_format_t format = input_format_t::json, const bool strict = true, const bool ignore_comments = false, - const bool ignore_trailing_commas = false) + const bool ignore_trailing_commas = false, + const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error) #if JSON_DELETE_DEPRECATED_FUNCTIONS = delete; #else @@ -5731,7 +5734,7 @@ public: // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) ? parser(std::move(ia), nullptr, true, ignore_comments, ignore_trailing_commas).sax_parse(sax, strict) // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) - : detail::binary_reader(std::move(ia), format).sax_parse(sax, strict); + : detail::binary_reader(std::move(ia), format).sax_parse(sax, strict, tag_handler); } #endif #if defined(__clang__) diff --git a/tests/src/unit-regression3.cpp b/tests/src/unit-regression3.cpp index 95b2b6d4f..12060c63d 100644 --- a/tests/src/unit-regression3.cpp +++ b/tests/src/unit-regression3.cpp @@ -862,6 +862,46 @@ TEST_CASE("issue #5338 - truncated CBOR tagged binary subtype is rejected") } } +TEST_CASE("issue #5676 - SAX parsing of CBOR tags") +{ + const json expected = json::binary({1, 2, 3}, 42); + const auto cbor = json::to_cbor(expected); + + nlohmann::detail::json_sax_acceptor acceptor; + CHECK_FALSE(json::sax_parse(cbor, &acceptor, json::input_format_t::cbor)); + CHECK_FALSE(json::sax_parse(cbor, &acceptor, json::input_format_t::cbor, + true, false, false, json::cbor_tag_handler_t::error)); + + CHECK(json::sax_parse(cbor, &acceptor, json::input_format_t::cbor, + true, false, false, json::cbor_tag_handler_t::ignore)); + + json parsed; + nlohmann::detail::json_sax_dom_parser sax(parsed); + CHECK(json::sax_parse(cbor, &sax, json::input_format_t::cbor, + true, false, false, json::cbor_tag_handler_t::store)); + CHECK(parsed == expected); + + json iterator_parsed; + nlohmann::detail::json_sax_dom_parser iterator_sax(iterator_parsed); + CHECK(json::sax_parse(cbor.begin(), cbor.end(), &iterator_sax, json::input_format_t::cbor, + true, false, false, json::cbor_tag_handler_t::store)); + CHECK(iterator_parsed == expected); + + json span_parsed; + nlohmann::detail::json_sax_dom_parser span_sax(span_parsed); + CHECK(json::sax_parse(nlohmann::detail::span_input_adapter(cbor.data(), cbor.size()), &span_sax, + json::input_format_t::cbor, true, false, false, json::cbor_tag_handler_t::store)); + CHECK(span_parsed == expected); + + const std::string text = "null"; + CHECK(json::sax_parse(text, &acceptor, json::input_format_t::json, + true, false, false, json::cbor_tag_handler_t::store)); + CHECK(json::sax_parse(text.begin(), text.end(), &acceptor, json::input_format_t::json, + true, false, false, json::cbor_tag_handler_t::store)); + CHECK(json::sax_parse(nlohmann::detail::span_input_adapter(text.data(), text.size()), &acceptor, + json::input_format_t::json, true, false, false, json::cbor_tag_handler_t::store)); +} + TEST_CASE("issue #5402 - update(merge_objects=true) overwrites a primitive with an object") { json t = {{"k", 1}}; From ff6f3d7d4af2b409c8efbb5af63441647c0131c8 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Wed, 7 Oct 2026 07:38:52 +0200 Subject: [PATCH 11/12] Reject nested indefinite-length CBOR string chunks (#5766) * fix(cbor): reject nested indefinite string chunks Signed-off-by: Joseph.Demarest Signed-off-by: Niels Lohmann * Address review comments on nested indefinite-length CBOR strings Rename is_chunk to inside_indefinite, update the stale test section names, and use lowercase comments like the surrounding code. Signed-off-by: Niels Lohmann --------- Signed-off-by: Joseph.Demarest Signed-off-by: Niels Lohmann Co-authored-by: Joseph.Demarest --- .../docs/features/binary_formats/cbor.md | 4 + .../nlohmann/detail/input/binary_reader.hpp | 90 ++++++++++--------- single_include/nlohmann/json.hpp | 90 ++++++++++--------- tests/src/unit-cbor.cpp | 39 ++++---- tests/src/unit-regression3.cpp | 8 ++ 5 files changed, 135 insertions(+), 96 deletions(-) diff --git a/docs/mkdocs/docs/features/binary_formats/cbor.md b/docs/mkdocs/docs/features/binary_formats/cbor.md index 1b226cd15..a0a0ed8d0 100644 --- a/docs/mkdocs/docs/features/binary_formats/cbor.md +++ b/docs/mkdocs/docs/features/binary_formats/cbor.md @@ -131,6 +131,7 @@ The library maps CBOR types to JSON value types as follows: | Byte string | binary | 0x59 | | Byte string | binary | 0x5A | | Byte string | binary | 0x5B | +| Byte string | binary | 0x5F | | UTF-8 string | string | 0x60..0x77 | | UTF-8 string | string | 0x78 | | UTF-8 string | string | 0x79 | @@ -156,6 +157,9 @@ The library maps CBOR types to JSON value types as follows: | Single-Precision Float | number_float | 0xFA | | Double-Precision Float | number_float | 0xFB | +Indefinite-length UTF-8 strings (0x7F) and byte strings (0x5F) are supported. Each chunk must be a definite-length +string of the same major type, as required by [RFC 8949, Section 3.2.3](https://www.rfc-editor.org/rfc/rfc8949.html#section-3.2.3). + !!! warning "Incomplete mapping" The mapping is **incomplete** in the sense that not all CBOR types can be converted to a JSON value. The following CBOR types are not supported and will yield parse errors: diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index d26e5fad4..3b0e3b021 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -1072,6 +1072,20 @@ class binary_reader } } + /*! + @brief reports a nested indefinite-length CBOR string or byte array + @param[in] type_name name of the rejected string type + @param[in] context parsing context for the error message + @return whether the SAX consumer accepts the parse error + */ + bool cbor_indefinite_string_error(const char* type_name, const char* context) + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, + exception_message(concat("indefinite-length ", type_name, + " is not allowed inside indefinite-length ", type_name, "; last byte: 0x", last_token), context), nullptr)); + } + /*! @brief reads a definite-length CBOR string @@ -1081,12 +1095,13 @@ class binary_reader into the same string. @param[out] result string the bytes are appended to + @param[in] inside_indefinite whether the bytes belong to an indefinite-length string @return whether string creation completed @pre @a current is not EOF */ - bool get_cbor_string_chunk(string_t& result) + bool get_cbor_string_chunk(string_t& result, const bool inside_indefinite) { switch (current) { @@ -1147,7 +1162,7 @@ class binary_reader { auto last_token = get_token_string(); return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, - exception_message(concat("expected length specification (0x60-0x7B) or indefinite string type (0x7F); last byte: 0x", last_token), "string"), nullptr)); + exception_message(concat("expected length specification (0x60-0x7B)", inside_indefinite ? "" : " or indefinite string type (0x7F)", "; last byte: 0x", last_token), "string"), nullptr)); } } } @@ -1165,13 +1180,9 @@ class binary_reader */ bool get_cbor_string(string_t& result, const char* context = "string") { - // number of indefinite-length strings that have been opened and not - // closed yet. RFC 8949, Section 3.2.3 does not permit nesting them, - // but this reader has always accepted it, so the open levels are - // counted instead of recursed through, which overflowed the stack for - // an input of repeated 0x7F bytes (see #5104). Every chunk is appended - // to the same result, so no per-level state is needed. - std::size_t open = 0; + // read chunks iteratively, but reject a second indefinite-length + // level as required by RFC 8949, Section 3.2.3 + bool indefinite = false; while (true) { @@ -1182,29 +1193,28 @@ class binary_reader if (current == 0x7F) // UTF-8 string (indefinite length) { - ++open; - get(); - continue; - } - - // a break marker closes the innermost indefinite-length string; - // outside of one it is not a string and falls through to the error - if (open != 0 && current == 0xFF) - { - if (--open == 0) + if (JSON_HEDLEY_UNLIKELY(indefinite)) { - return check_string_utf8(result, context); + return cbor_indefinite_string_error("string", "string"); } + indefinite = true; get(); continue; } - if (JSON_HEDLEY_UNLIKELY(!get_cbor_string_chunk(result))) + // a break marker closes the indefinite-length string; outside + // of one it falls through to the error below + if (indefinite && current == 0xFF) + { + return check_string_utf8(result, context); + } + + if (JSON_HEDLEY_UNLIKELY(!get_cbor_string_chunk(result, indefinite))) { return false; } - if (open == 0) + if (!indefinite) { return check_string_utf8(result, context); } @@ -1296,12 +1306,13 @@ class binary_reader read into the same byte array. @param[out] result byte array the bytes are appended to + @param[in] inside_indefinite whether the bytes belong to an indefinite-length string @return whether byte array creation completed @pre @a current is not EOF */ - bool get_cbor_binary_chunk(binary_t& result) + bool get_cbor_binary_chunk(binary_t& result, const bool inside_indefinite) { switch (current) { @@ -1366,7 +1377,7 @@ class binary_reader { auto last_token = get_token_string(); return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, - exception_message(concat("expected length specification (0x40-0x5B) or indefinite binary array type (0x5F); last byte: 0x", last_token), "binary"), nullptr)); + exception_message(concat("expected length specification (0x40-0x5B)", inside_indefinite ? "" : " or indefinite binary array type (0x5F)", "; last byte: 0x", last_token), "binary"), nullptr)); } } } @@ -1384,9 +1395,9 @@ class binary_reader */ bool get_cbor_binary(binary_t& result) { - // the open indefinite-length byte arrays are counted rather than - // recursed through, for the reason given in @ref get_cbor_string - std::size_t open = 0; + // read chunks iteratively, but reject a second indefinite-length + // level as required by RFC 8949, Section 3.2.3 + bool indefinite = false; while (true) { @@ -1397,29 +1408,28 @@ class binary_reader if (current == 0x5F) // Binary data (indefinite length) { - ++open; - get(); - continue; - } - - // a break marker closes the innermost indefinite-length byte - // array; outside of one it falls through to the error below - if (open != 0 && current == 0xFF) - { - if (--open == 0) + if (JSON_HEDLEY_UNLIKELY(indefinite)) { - return true; + return cbor_indefinite_string_error("binary array", "binary"); } + indefinite = true; get(); continue; } - if (JSON_HEDLEY_UNLIKELY(!get_cbor_binary_chunk(result))) + // a break marker closes the indefinite-length string; outside + // of one it falls through to the error below + if (indefinite && current == 0xFF) + { + return true; + } + + if (JSON_HEDLEY_UNLIKELY(!get_cbor_binary_chunk(result, indefinite))) { return false; } - if (open == 0) + if (!indefinite) { return true; } diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 7d19b53af..f20e6051c 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -14768,6 +14768,20 @@ class binary_reader } } + /*! + @brief reports a nested indefinite-length CBOR string or byte array + @param[in] type_name name of the rejected string type + @param[in] context parsing context for the error message + @return whether the SAX consumer accepts the parse error + */ + bool cbor_indefinite_string_error(const char* type_name, const char* context) + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, + exception_message(concat("indefinite-length ", type_name, + " is not allowed inside indefinite-length ", type_name, "; last byte: 0x", last_token), context), nullptr)); + } + /*! @brief reads a definite-length CBOR string @@ -14777,12 +14791,13 @@ class binary_reader into the same string. @param[out] result string the bytes are appended to + @param[in] inside_indefinite whether the bytes belong to an indefinite-length string @return whether string creation completed @pre @a current is not EOF */ - bool get_cbor_string_chunk(string_t& result) + bool get_cbor_string_chunk(string_t& result, const bool inside_indefinite) { switch (current) { @@ -14843,7 +14858,7 @@ class binary_reader { auto last_token = get_token_string(); return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, - exception_message(concat("expected length specification (0x60-0x7B) or indefinite string type (0x7F); last byte: 0x", last_token), "string"), nullptr)); + exception_message(concat("expected length specification (0x60-0x7B)", inside_indefinite ? "" : " or indefinite string type (0x7F)", "; last byte: 0x", last_token), "string"), nullptr)); } } } @@ -14861,13 +14876,9 @@ class binary_reader */ bool get_cbor_string(string_t& result, const char* context = "string") { - // number of indefinite-length strings that have been opened and not - // closed yet. RFC 8949, Section 3.2.3 does not permit nesting them, - // but this reader has always accepted it, so the open levels are - // counted instead of recursed through, which overflowed the stack for - // an input of repeated 0x7F bytes (see #5104). Every chunk is appended - // to the same result, so no per-level state is needed. - std::size_t open = 0; + // read chunks iteratively, but reject a second indefinite-length + // level as required by RFC 8949, Section 3.2.3 + bool indefinite = false; while (true) { @@ -14878,29 +14889,28 @@ class binary_reader if (current == 0x7F) // UTF-8 string (indefinite length) { - ++open; - get(); - continue; - } - - // a break marker closes the innermost indefinite-length string; - // outside of one it is not a string and falls through to the error - if (open != 0 && current == 0xFF) - { - if (--open == 0) + if (JSON_HEDLEY_UNLIKELY(indefinite)) { - return check_string_utf8(result, context); + return cbor_indefinite_string_error("string", "string"); } + indefinite = true; get(); continue; } - if (JSON_HEDLEY_UNLIKELY(!get_cbor_string_chunk(result))) + // a break marker closes the indefinite-length string; outside + // of one it falls through to the error below + if (indefinite && current == 0xFF) + { + return check_string_utf8(result, context); + } + + if (JSON_HEDLEY_UNLIKELY(!get_cbor_string_chunk(result, indefinite))) { return false; } - if (open == 0) + if (!indefinite) { return check_string_utf8(result, context); } @@ -14992,12 +15002,13 @@ class binary_reader read into the same byte array. @param[out] result byte array the bytes are appended to + @param[in] inside_indefinite whether the bytes belong to an indefinite-length string @return whether byte array creation completed @pre @a current is not EOF */ - bool get_cbor_binary_chunk(binary_t& result) + bool get_cbor_binary_chunk(binary_t& result, const bool inside_indefinite) { switch (current) { @@ -15062,7 +15073,7 @@ class binary_reader { auto last_token = get_token_string(); return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, - exception_message(concat("expected length specification (0x40-0x5B) or indefinite binary array type (0x5F); last byte: 0x", last_token), "binary"), nullptr)); + exception_message(concat("expected length specification (0x40-0x5B)", inside_indefinite ? "" : " or indefinite binary array type (0x5F)", "; last byte: 0x", last_token), "binary"), nullptr)); } } } @@ -15080,9 +15091,9 @@ class binary_reader */ bool get_cbor_binary(binary_t& result) { - // the open indefinite-length byte arrays are counted rather than - // recursed through, for the reason given in @ref get_cbor_string - std::size_t open = 0; + // read chunks iteratively, but reject a second indefinite-length + // level as required by RFC 8949, Section 3.2.3 + bool indefinite = false; while (true) { @@ -15093,29 +15104,28 @@ class binary_reader if (current == 0x5F) // Binary data (indefinite length) { - ++open; - get(); - continue; - } - - // a break marker closes the innermost indefinite-length byte - // array; outside of one it falls through to the error below - if (open != 0 && current == 0xFF) - { - if (--open == 0) + if (JSON_HEDLEY_UNLIKELY(indefinite)) { - return true; + return cbor_indefinite_string_error("binary array", "binary"); } + indefinite = true; get(); continue; } - if (JSON_HEDLEY_UNLIKELY(!get_cbor_binary_chunk(result))) + // a break marker closes the indefinite-length string; outside + // of one it falls through to the error below + if (indefinite && current == 0xFF) + { + return true; + } + + if (JSON_HEDLEY_UNLIKELY(!get_cbor_binary_chunk(result, indefinite))) { return false; } - if (open == 0) + if (!indefinite) { return true; } diff --git a/tests/src/unit-cbor.cpp b/tests/src/unit-cbor.cpp index 38763b4db..9ee371495 100644 --- a/tests/src/unit-cbor.cpp +++ b/tests/src/unit-cbor.cpp @@ -1699,7 +1699,7 @@ TEST_CASE("CBOR") CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector({0xA1, 0x61, 0X61})), "[json.exception.parse_error.110] parse error at byte 4: syntax error while parsing CBOR value: unexpected end of input", json::parse_error&); CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector({0xBF, 0x61, 0X61})), "[json.exception.parse_error.110] parse error at byte 4: syntax error while parsing CBOR value: unexpected end of input", json::parse_error&); CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector({0x5F})), "[json.exception.parse_error.110] parse error at byte 2: syntax error while parsing CBOR binary: unexpected end of input", json::parse_error&); - CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector({0x5F, 0x00})), "[json.exception.parse_error.113] parse error at byte 2: syntax error while parsing CBOR binary: expected length specification (0x40-0x5B) or indefinite binary array type (0x5F); last byte: 0x00", json::parse_error&); + CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector({0x5F, 0x00})), "[json.exception.parse_error.113] parse error at byte 2: syntax error while parsing CBOR binary: expected length specification (0x40-0x5B); last byte: 0x00", json::parse_error&); CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector({0x41})), "[json.exception.parse_error.110] parse error at byte 2: syntax error while parsing CBOR binary: unexpected end of input", json::parse_error&); CHECK(json::from_cbor(std::vector({0x18}), true, false).is_discarded()); @@ -2305,22 +2305,21 @@ TEST_CASE("CBOR indefinite-length strings do not recurse per chunk") { // Reading an indefinite-length string or byte array used to call itself // once per chunk, so a payload of repeated 0x7F (or 0x5F) bytes exhausted - // the call stack before any of the input was rejected. The open levels are - // counted now, and the levels below prove the reader still reads the same - // values and reports the same errors at the same byte offsets. + // the call stack before any of the input was rejected. Nested indefinite + // chunks are now rejected at the second byte, without recursing. json _; - SECTION("many open levels are reported, not crashed on") + SECTION("nested levels are rejected, not crashed on") { const std::vector input(200000, 0x7F); - CHECK_THROWS_WITH_AS(_ = json::from_cbor(input), "[json.exception.parse_error.110] parse error at byte 200001: syntax error while parsing CBOR string: unexpected end of input", json::parse_error&); + CHECK_THROWS_WITH_AS(_ = json::from_cbor(input), "[json.exception.parse_error.113] parse error at byte 2: syntax error while parsing CBOR string: indefinite-length string is not allowed inside indefinite-length string; last byte: 0x7F", json::parse_error&); CHECK(json::from_cbor(input, true, false).is_discarded()); } - SECTION("many open levels are reported, not crashed on (binary)") + SECTION("nested levels are rejected, not crashed on (binary)") { const std::vector input(200000, 0x5F); - CHECK_THROWS_WITH_AS(_ = json::from_cbor(input), "[json.exception.parse_error.110] parse error at byte 200001: syntax error while parsing CBOR binary: unexpected end of input", json::parse_error&); + CHECK_THROWS_WITH_AS(_ = json::from_cbor(input), "[json.exception.parse_error.113] parse error at byte 2: syntax error while parsing CBOR binary: indefinite-length binary array is not allowed inside indefinite-length binary array; last byte: 0x5F", json::parse_error&); CHECK(json::from_cbor(input, true, false).is_discarded()); } @@ -2328,22 +2327,22 @@ TEST_CASE("CBOR indefinite-length strings do not recurse per chunk") { CHECK(json::from_cbor(std::vector({0x7F, 0xFF})) == json("")); CHECK(json::from_cbor(std::vector({0x7F, 0x61, 0x61, 0xFF})) == json("a")); - // nested indefinite-length strings are concatenated across levels - CHECK(json::from_cbor(std::vector({0x7F, 0x7F, 0x61, 0x61, 0xFF, 0x61, 0x62, 0xFF})) == json("ab")); - CHECK(json::from_cbor(std::vector({0x7F, 0x7F, 0x7F, 0x61, 0x7A, 0xFF, 0xFF, 0xFF})) == json("z")); + // empty and nonempty definite-length chunks concatenate in order + CHECK(json::from_cbor(std::vector({0x7F, 0x61, 'a', 0x60, 0x61, 'b', 0x61, 'c', 0xFF})) == json("abc")); CHECK(json::from_cbor(std::vector({0xA1, 0x7F, 0x61, 0x61, 0xFF, 0x01})) == json({{"a", 1}})); } SECTION("chunks are still concatenated (binary)") { CHECK(json::from_cbor(std::vector({0x5F, 0x41, 0x61, 0xFF})) == json::binary({0x61})); - CHECK(json::from_cbor(std::vector({0x5F, 0x5F, 0x41, 0x61, 0xFF, 0x41, 0x62, 0xFF})) == json::binary({0x61, 0x62})); + CHECK(json::from_cbor(std::vector({0x5F, 0xFF})) == json::binary({})); + CHECK(json::from_cbor(std::vector({0x5F, 0x41, 0x61, 0x40, 0x41, 0x62, 0x41, 0x63, 0xFF})) == json::binary({0x61, 0x62, 0x63})); } SECTION("a chunk that is not a string is still rejected") { - CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector({0x7F, 0x7F, 0x00})), "[json.exception.parse_error.113] parse error at byte 3: syntax error while parsing CBOR string: expected length specification (0x60-0x7B) or indefinite string type (0x7F); last byte: 0x00", json::parse_error&); - CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector({0x5F, 0x5F, 0x00})), "[json.exception.parse_error.113] parse error at byte 3: syntax error while parsing CBOR binary: expected length specification (0x40-0x5B) or indefinite binary array type (0x5F); last byte: 0x00", json::parse_error&); + CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector({0x7F, 0x00})), "[json.exception.parse_error.113] parse error at byte 2: syntax error while parsing CBOR string: expected length specification (0x60-0x7B); last byte: 0x00", json::parse_error&); + CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector({0x5F, 0x00})), "[json.exception.parse_error.113] parse error at byte 2: syntax error while parsing CBOR binary: expected length specification (0x40-0x5B); last byte: 0x00", json::parse_error&); } SECTION("a break marker outside an indefinite-length string is not a string") @@ -2896,9 +2895,17 @@ TEST_CASE("examples from RFC 8949 Appendix A") { const auto packed = utils::read_binary_file(TEST_DATA_DIRECTORY "/binary_data/cbor_binary.cbor"); json j; - CHECK_NOTHROW(j = json::from_cbor(packed)); + // the fixture's tail contains nested indefinite-length byte strings. + CHECK_THROWS_WITH_AS(j = json::from_cbor(packed), "[json.exception.parse_error.113] parse error at byte 513: syntax error while parsing CBOR binary: indefinite-length binary array is not allowed inside indefinite-length binary array; last byte: 0x5F", json::parse_error&); - const auto expected = utils::read_binary_file(TEST_DATA_DIRECTORY "/binary_data/cbor_binary.out"); + // keep the byte-for-byte decoding check for its valid prefix: the first + // 512 encoded bytes contain 468 payload bytes in definite-length chunks. + auto valid_prefix = packed; + valid_prefix.resize(512); + valid_prefix.push_back(0xFF); + auto expected = utils::read_binary_file(TEST_DATA_DIRECTORY "/binary_data/cbor_binary.out"); + expected.resize(468); + CHECK_NOTHROW(j = json::from_cbor(valid_prefix)); CHECK(j == json::binary(expected)); // 0xd8 diff --git a/tests/src/unit-regression3.cpp b/tests/src/unit-regression3.cpp index 12060c63d..d4f75a36e 100644 --- a/tests/src/unit-regression3.cpp +++ b/tests/src/unit-regression3.cpp @@ -960,4 +960,12 @@ TEST_CASE("regression test #5476 - array type without reserve()") } } +TEST_CASE("issue #5317 - nested indefinite-length CBOR string chunks are rejected") +{ + json _; + CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector({0x7F, 0x7F, 0x61, 0x61, 0xFF, 0xFF})), "[json.exception.parse_error.113] parse error at byte 2: syntax error while parsing CBOR string: indefinite-length string is not allowed inside indefinite-length string; last byte: 0x7F", json::parse_error&); + CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector({0x5F, 0x5F, 0x41, 0x61, 0xFF, 0xFF})), "[json.exception.parse_error.113] parse error at byte 2: syntax error while parsing CBOR binary: indefinite-length binary array is not allowed inside indefinite-length binary array; last byte: 0x5F", json::parse_error&); + CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector({0xA1, 0x7F, 0x7F, 0xFF, 0xFF, 0x01})), "[json.exception.parse_error.113] parse error at byte 3: syntax error while parsing CBOR string: indefinite-length string is not allowed inside indefinite-length string; last byte: 0x7F", json::parse_error&); +} + DOCTEST_CLANG_SUPPRESS_WARNING_POP From 1c754cfe31e353ea5eb85bd3aeca831996398ec8 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Wed, 7 Oct 2026 07:39:05 +0200 Subject: [PATCH 12/12] Copy a pair-shaped array value under JSON_BRACE_INIT_COPY_SEMANTICS (#5701) With JSON_BRACE_INIT_COPY_SEMANTICS enabled, single-element brace initialization from a JSON value decided whether to copy the value or build an object by inspecting the value's runtime shape: a two-element array whose first element is a string, such as ["key", 42], was turned into an object instead of being copied. This made the behavior depend on the element's content, and it did not distinguish an existing value of this shape from a nested braced pair written in the source, such as the inner {"key", "value"} of {{"key", "value"}}. json_ref now records whether it was constructed from a braced list (true only for the std::initializer_list constructor used for nested braced lists) or from a value. The initializer-list constructor uses this to copy or move a single non-braced-list element before deciding whether the list describes an object, so a JSON value is always copied regardless of its shape, while a braced pair written in the source still creates an object. Fixes #5662. Signed-off-by: Niels Lohmann --- .../macros/json_brace_init_copy_semantics.md | 8 ++++--- include/nlohmann/detail/json_ref.hpp | 9 ++++++++ include/nlohmann/json.hpp | 12 ++++++++++ single_include/nlohmann/json.hpp | 21 ++++++++++++++++++ tests/src/unit-brace-init-copy-semantics.cpp | 22 +++++++++++++++++++ 5 files changed, 69 insertions(+), 3 deletions(-) diff --git a/docs/mkdocs/docs/api/macros/json_brace_init_copy_semantics.md b/docs/mkdocs/docs/api/macros/json_brace_init_copy_semantics.md index 71b8bb28f..a038f8435 100644 --- a/docs/mkdocs/docs/api/macros/json_brace_init_copy_semantics.md +++ b/docs/mkdocs/docs/api/macros/json_brace_init_copy_semantics.md @@ -50,9 +50,11 @@ The default value is `0` (disabled — existing behavior is preserved). ``` Code that relies on these producing arrays must use `json::array()` instead (see below). Lists with more than one - element, and a single `[string, value]` pair such as `{{"key", "value"}}`, which still creates an object, are not - affected. The library's own conversions are not affected either: for example, `std::tuple{5}` still becomes - `[5]`. + element, and a single `[string, value]` pair *written as a braced list*, such as `{{"key", "value"}}`, which still + creates an object, are not affected. This exception is based on how the pair is written, not on the shape of its + value: an existing JSON value that happens to be a two-element array with a string as its first element, such as + `json arr = {"key", 42};`, is still copied by `json j{arr};` rather than turned into an object. The library's own + conversions are not affected either: for example, `std::tuple{5}` still becomes `[5]`. !!! note "ABI compatibility" diff --git a/include/nlohmann/detail/json_ref.hpp b/include/nlohmann/detail/json_ref.hpp index 0d52b0a62..b270c9c54 100644 --- a/include/nlohmann/detail/json_ref.hpp +++ b/include/nlohmann/detail/json_ref.hpp @@ -34,6 +34,7 @@ class json_ref json_ref(std::initializer_list init) : owned_value(init) + , braced_list(true) {} template < @@ -69,9 +70,17 @@ class json_ref return &** this; } + /// whether the value was written as a braced list, such as {"key", 1}, + /// rather than given as a value + bool is_braced_list() const noexcept + { + return braced_list; + } + private: mutable value_type owned_value = nullptr; value_type const* value_ref = nullptr; + bool braced_list = false; }; } // namespace detail diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 397b253ad..9c89ab0bc 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -2252,6 +2252,18 @@ public: bool type_deduction = true, value_t manual_type = value_t::array) { +#if JSON_BRACE_INIT_COPY_SEMANTICS + // a single element that is a value rather than a braced list is + // copied or moved as is, whatever its content looks like + if (type_deduction && init.size() == 1 && !init.begin()->is_braced_list()) + { + *this = init.begin()->moved_or_copied(); + set_parents(); + assert_invariant(); + return; + } +#endif + // check if each element is an array with two elements whose first // element is a string bool is_an_object = std::all_of(init.begin(), init.end(), diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index f20e6051c..431f76c02 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -21098,6 +21098,7 @@ class json_ref json_ref(std::initializer_list init) : owned_value(init) + , braced_list(true) {} template < @@ -21133,9 +21134,17 @@ class json_ref return &** this; } + /// whether the value was written as a braced list, such as {"key", 1}, + /// rather than given as a value + bool is_braced_list() const noexcept + { + return braced_list; + } + private: mutable value_type owned_value = nullptr; value_type const* value_ref = nullptr; + bool braced_list = false; }; } // namespace detail @@ -29454,6 +29463,18 @@ public: bool type_deduction = true, value_t manual_type = value_t::array) { +#if JSON_BRACE_INIT_COPY_SEMANTICS + // a single element that is a value rather than a braced list is + // copied or moved as is, whatever its content looks like + if (type_deduction && init.size() == 1 && !init.begin()->is_braced_list()) + { + *this = init.begin()->moved_or_copied(); + set_parents(); + assert_invariant(); + return; + } +#endif + // check if each element is an array with two elements whose first // element is a string bool is_an_object = std::all_of(init.begin(), init.end(), diff --git a/tests/src/unit-brace-init-copy-semantics.cpp b/tests/src/unit-brace-init-copy-semantics.cpp index 1ee0c6607..e3406758b 100644 --- a/tests/src/unit-brace-init-copy-semantics.cpp +++ b/tests/src/unit-brace-init-copy-semantics.cpp @@ -72,6 +72,28 @@ TEST_CASE("JSON_BRACE_INIT_COPY_SEMANTICS") CHECK(j7 == json::array({1, 2})); } + SECTION("single-element brace initialization copies a pair-shaped array value (#5662)") + { + // a JSON value that happens to be a 2-element array whose first + // element is a string must still be copied, not turned into an + // object; only a braced list written in the source, such as the + // inner {"key", "value"} of {{"key", "value"}}, describes an object + json const pair_shaped = json::array({"key", 42}); + + json const j1{pair_shaped}; + CHECK(j1.is_array()); + CHECK(j1 == pair_shaped); + + json const j2 = {pair_shaped}; + CHECK(j2.is_array()); + CHECK(j2 == pair_shaped); + + // the same holds for an rvalue of the same shape + json const j3{json::array({"key", 42})}; + CHECK(j3.is_array()); + CHECK(j3 == pair_shaped); + } + SECTION("what the macro does not change") { // lists with more than one element are unaffected