From 6d0867a85f0148ae184f457ec6755f9d4e184132 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Fri, 2 Oct 2026 17:25:17 +0200 Subject: [PATCH 01/22] Make comparisons with scalars noexcept only when the conversion is (#5751) The comparison operators taking a scalar (==, !=, <, <=, >, >=, and C++20's <=>) convert the scalar to a basic_json and compare, but were unconditionally noexcept. When that conversion throws, the program called std::terminate instead of propagating the exception, e.g. when comparing a json with a string literal under memory pressure (std::bad_alloc) or with an enum value not mapped by NLOHMANN_JSON_SERIALIZE_ENUM_STRICT (out_of_range.410). clang-tidy 22.1 reports the latter as bugprone-exception-escape. Declare the 16 scalar overloads noexcept(std::is_nothrow_constructible::value): they stay noexcept for numbers, Booleans, nullptr, and plain enums, and are noexcept(false) for strings and enums whose to_json may throw. The comparisons of two basic_json values are unchanged. Restore the strict-enum comparisons removed from unit-conversions.cpp in the previous PR, check that comparing an unmapped strict enum now throws, and pin the new exception specifications in unit-noexcept.cpp. Document the exception safety of overload (2) on all seven operator pages. Ran make amalgamate. Signed-off-by: Niels Lohmann --- .../mkdocs/docs/api/basic_json/operator_eq.md | 14 +++++--- .../mkdocs/docs/api/basic_json/operator_ge.md | 12 +++++-- .../mkdocs/docs/api/basic_json/operator_gt.md | 12 +++++-- .../mkdocs/docs/api/basic_json/operator_le.md | 12 +++++-- .../mkdocs/docs/api/basic_json/operator_lt.md | 12 +++++-- .../mkdocs/docs/api/basic_json/operator_ne.md | 12 +++++-- .../docs/api/basic_json/operator_spaceship.md | 10 ++++-- include/nlohmann/json.hpp | 32 +++++++++---------- single_include/nlohmann/json.hpp | 32 +++++++++---------- tests/src/unit-conversions.cpp | 21 ++++++++++++ tests/src/unit-noexcept.cpp | 9 ++++++ 11 files changed, 125 insertions(+), 53 deletions(-) diff --git a/docs/mkdocs/docs/api/basic_json/operator_eq.md b/docs/mkdocs/docs/api/basic_json/operator_eq.md index 26eda720f..58a7e74da 100644 --- a/docs/mkdocs/docs/api/basic_json/operator_eq.md +++ b/docs/mkdocs/docs/api/basic_json/operator_eq.md @@ -5,17 +5,17 @@ bool operator==(const_reference lhs, const_reference rhs) noexcept; // (1) template -bool operator==(const_reference lhs, const ScalarType rhs) noexcept; // (2) +bool operator==(const_reference lhs, const ScalarType rhs) noexcept(/* see below */); // (2) template -bool operator==(ScalarType lhs, const const_reference rhs) noexcept; // (2) +bool operator==(ScalarType lhs, const const_reference rhs) noexcept(/* see below */); // (2) // since C++20 class basic_json { bool operator==(const_reference rhs) const noexcept; // (1) template - bool operator==(ScalarType rhs) const noexcept; // (2) + bool operator==(ScalarType rhs) const noexcept(/* see below */); // (2) }; ``` @@ -46,7 +46,12 @@ whether the values `lhs`/`*this` and `rhs` are equal ## Exception safety -No-throw guarantee: this function never throws exceptions. +1. No-throw guarantee: this function never throws exceptions. +2. No-throw guarantee if converting the scalar to a JSON value cannot throw, as for numbers, Booleans, and + `#!cpp nullptr`; the function is `#!cpp noexcept` exactly in that case. Otherwise, it throws what the conversion + throws, for example `std::bad_alloc` when converting a string, or + [`out_of_range.410`](../../home/exceptions.md#jsonexceptionout_of_range410) for an enum value not mapped by + [`NLOHMANN_JSON_SERIALIZE_ENUM_STRICT`](../macros/nlohmann_json_serialize_enum_strict.md). ## Complexity @@ -171,3 +176,4 @@ Linear. 1. Added in version 1.0.0. Added C++20 member functions in version 3.11.0. 2. Added in version 1.0.0. Added C++20 member functions in version 3.11.0. + Made conditionally `#!cpp noexcept` in version 3.13.0; before, a throwing conversion called `std::terminate`. diff --git a/docs/mkdocs/docs/api/basic_json/operator_ge.md b/docs/mkdocs/docs/api/basic_json/operator_ge.md index f7899beab..99949f226 100644 --- a/docs/mkdocs/docs/api/basic_json/operator_ge.md +++ b/docs/mkdocs/docs/api/basic_json/operator_ge.md @@ -5,10 +5,10 @@ bool operator>=(const_reference lhs, const_reference rhs) noexcept; // (1) template -bool operator>=(const_reference lhs, const ScalarType rhs) noexcept; // (2) +bool operator>=(const_reference lhs, const ScalarType rhs) noexcept(/* see below */); // (2) template -bool operator>=(ScalarType lhs, const const_reference rhs) noexcept; // (2) +bool operator>=(ScalarType lhs, const const_reference rhs) noexcept(/* see below */); // (2) ``` 1. Compares whether one JSON value `lhs` is greater than or equal to another JSON value `rhs` according to the following @@ -39,7 +39,12 @@ whether `lhs` is greater than or equal to `rhs` ## Exception safety -No-throw guarantee: this function never throws exceptions. +1. No-throw guarantee: this function never throws exceptions. +2. No-throw guarantee if converting the scalar to a JSON value cannot throw, as for numbers, Booleans, and + `#!cpp nullptr`; the function is `#!cpp noexcept` exactly in that case. Otherwise, it throws what the conversion + throws, for example `std::bad_alloc` when converting a string, or + [`out_of_range.410`](../../home/exceptions.md#jsonexceptionout_of_range410) for an enum value not mapped by + [`NLOHMANN_JSON_SERIALIZE_ENUM_STRICT`](../macros/nlohmann_json_serialize_enum_strict.md). ## Complexity @@ -94,3 +99,4 @@ Linear. 1. Added in version 1.0.0. Conditionally removed since C++20 in version 3.11.0. 2. Added in version 1.0.0. Conditionally removed since C++20 in version 3.11.0. + Made conditionally `#!cpp noexcept` in version 3.13.0; before, a throwing conversion called `std::terminate`. diff --git a/docs/mkdocs/docs/api/basic_json/operator_gt.md b/docs/mkdocs/docs/api/basic_json/operator_gt.md index 486da5fd0..3fcb339da 100644 --- a/docs/mkdocs/docs/api/basic_json/operator_gt.md +++ b/docs/mkdocs/docs/api/basic_json/operator_gt.md @@ -5,10 +5,10 @@ bool operator>(const_reference lhs, const_reference rhs) noexcept; // (1) template -bool operator>(const_reference lhs, const ScalarType rhs) noexcept; // (2) +bool operator>(const_reference lhs, const ScalarType rhs) noexcept(/* see below */); // (2) template -bool operator>(ScalarType lhs, const const_reference rhs) noexcept; // (2) +bool operator>(ScalarType lhs, const const_reference rhs) noexcept(/* see below */); // (2) ``` 1. Compares whether one JSON value `lhs` is greater than another JSON value `rhs` according to the @@ -39,7 +39,12 @@ whether `lhs` is greater than `rhs` ## Exception safety -No-throw guarantee: this function never throws exceptions. +1. No-throw guarantee: this function never throws exceptions. +2. No-throw guarantee if converting the scalar to a JSON value cannot throw, as for numbers, Booleans, and + `#!cpp nullptr`; the function is `#!cpp noexcept` exactly in that case. Otherwise, it throws what the conversion + throws, for example `std::bad_alloc` when converting a string, or + [`out_of_range.410`](../../home/exceptions.md#jsonexceptionout_of_range410) for an enum value not mapped by + [`NLOHMANN_JSON_SERIALIZE_ENUM_STRICT`](../macros/nlohmann_json_serialize_enum_strict.md). ## Complexity @@ -84,3 +89,4 @@ Linear. 1. Added in version 1.0.0. Conditionally removed since C++20 in version 3.11.0. 2. Added in version 1.0.0. Conditionally removed since C++20 in version 3.11.0. + Made conditionally `#!cpp noexcept` in version 3.13.0; before, a throwing conversion called `std::terminate`. diff --git a/docs/mkdocs/docs/api/basic_json/operator_le.md b/docs/mkdocs/docs/api/basic_json/operator_le.md index 4334fe35e..db193d0db 100644 --- a/docs/mkdocs/docs/api/basic_json/operator_le.md +++ b/docs/mkdocs/docs/api/basic_json/operator_le.md @@ -5,10 +5,10 @@ bool operator<=(const_reference lhs, const_reference rhs) noexcept; // (1) template -bool operator<=(const_reference lhs, const ScalarType rhs) noexcept; // (2) +bool operator<=(const_reference lhs, const ScalarType rhs) noexcept(/* see below */); // (2) template -bool operator<=(ScalarType lhs, const const_reference rhs) noexcept; // (2) +bool operator<=(ScalarType lhs, const const_reference rhs) noexcept(/* see below */); // (2) ``` 1. Compares whether one JSON value `lhs` is less than or equal to another JSON value `rhs` @@ -40,7 +40,12 @@ whether `lhs` is less than or equal to `rhs` ## Exception safety -No-throw guarantee: this function never throws exceptions. +1. No-throw guarantee: this function never throws exceptions. +2. No-throw guarantee if converting the scalar to a JSON value cannot throw, as for numbers, Booleans, and + `#!cpp nullptr`; the function is `#!cpp noexcept` exactly in that case. Otherwise, it throws what the conversion + throws, for example `std::bad_alloc` when converting a string, or + [`out_of_range.410`](../../home/exceptions.md#jsonexceptionout_of_range410) for an enum value not mapped by + [`NLOHMANN_JSON_SERIALIZE_ENUM_STRICT`](../macros/nlohmann_json_serialize_enum_strict.md). ## Complexity @@ -95,3 +100,4 @@ Linear. 1. Added in version 1.0.0. Conditionally removed since C++20 in version 3.11.0. 2. Added in version 1.0.0. Conditionally removed since C++20 in version 3.11.0. + Made conditionally `#!cpp noexcept` in version 3.13.0; before, a throwing conversion called `std::terminate`. diff --git a/docs/mkdocs/docs/api/basic_json/operator_lt.md b/docs/mkdocs/docs/api/basic_json/operator_lt.md index 118d817c8..1b8e65225 100644 --- a/docs/mkdocs/docs/api/basic_json/operator_lt.md +++ b/docs/mkdocs/docs/api/basic_json/operator_lt.md @@ -5,10 +5,10 @@ bool operator<(const_reference lhs, const_reference rhs) noexcept; // (1) template -bool operator<(const_reference lhs, const ScalarType rhs) noexcept; // (2) +bool operator<(const_reference lhs, const ScalarType rhs) noexcept(/* see below */); // (2) template -bool operator<(ScalarType lhs, const const_reference rhs) noexcept; // (2) +bool operator<(ScalarType lhs, const const_reference rhs) noexcept(/* see below */); // (2) ``` 1. Compares whether one JSON value `lhs` is less than another JSON value `rhs` according to the @@ -49,7 +49,12 @@ whether `lhs` is less than `rhs` ## Exception safety -No-throw guarantee: this function never throws exceptions. +1. No-throw guarantee: this function never throws exceptions. +2. No-throw guarantee if converting the scalar to a JSON value cannot throw, as for numbers, Booleans, and + `#!cpp nullptr`; the function is `#!cpp noexcept` exactly in that case. Otherwise, it throws what the conversion + throws, for example `std::bad_alloc` when converting a string, or + [`out_of_range.410`](../../home/exceptions.md#jsonexceptionout_of_range410) for an enum value not mapped by + [`NLOHMANN_JSON_SERIALIZE_ENUM_STRICT`](../macros/nlohmann_json_serialize_enum_strict.md). ## Complexity @@ -94,3 +99,4 @@ Linear. 1. Added in version 1.0.0. Conditionally removed since C++20 in version 3.11.0. 2. Added in version 1.0.0. Conditionally removed since C++20 in version 3.11.0. + Made conditionally `#!cpp noexcept` in version 3.13.0; before, a throwing conversion called `std::terminate`. diff --git a/docs/mkdocs/docs/api/basic_json/operator_ne.md b/docs/mkdocs/docs/api/basic_json/operator_ne.md index a8c6fecc2..ceeb31ab3 100644 --- a/docs/mkdocs/docs/api/basic_json/operator_ne.md +++ b/docs/mkdocs/docs/api/basic_json/operator_ne.md @@ -5,10 +5,10 @@ bool operator!=(const_reference lhs, const_reference rhs) noexcept; // (1) template -bool operator!=(const_reference lhs, const ScalarType rhs) noexcept; // (2) +bool operator!=(const_reference lhs, const ScalarType rhs) noexcept(/* see below */); // (2) template -bool operator!=(ScalarType lhs, const const_reference rhs) noexcept; // (2) +bool operator!=(ScalarType lhs, const const_reference rhs) noexcept(/* see below */); // (2) ``` 1. Compares two JSON values for inequality. Returns `#!cpp !(lhs == rhs)`. @@ -36,7 +36,12 @@ whether the values `lhs`/`*this` and `rhs` are not equal ## Exception safety -No-throw guarantee: this function never throws exceptions. +1. No-throw guarantee: this function never throws exceptions. +2. No-throw guarantee if converting the scalar to a JSON value cannot throw, as for numbers, Booleans, and + `#!cpp nullptr`; the function is `#!cpp noexcept` exactly in that case. Otherwise, it throws what the conversion + throws, for example `std::bad_alloc` when converting a string, or + [`out_of_range.410`](../../home/exceptions.md#jsonexceptionout_of_range410) for an enum value not mapped by + [`NLOHMANN_JSON_SERIALIZE_ENUM_STRICT`](../macros/nlohmann_json_serialize_enum_strict.md). ## Complexity @@ -98,3 +103,4 @@ Linear. member function in version 3.13.0; since C++20, the compiler rewrites `a != b` using `operator==`. 2. Added in version 1.0.0. Changed in version 3.13.0 to remove special-casing for `NaN` and `discarded` values; `operator!=` now consistently means `!(a == b)`. Since C++20, the compiler rewrites `a != b` using `operator==`. + Made conditionally `#!cpp noexcept` in version 3.13.0; before, a throwing conversion called `std::terminate`. diff --git a/docs/mkdocs/docs/api/basic_json/operator_spaceship.md b/docs/mkdocs/docs/api/basic_json/operator_spaceship.md index 9e91d0d2d..47ca22484 100644 --- a/docs/mkdocs/docs/api/basic_json/operator_spaceship.md +++ b/docs/mkdocs/docs/api/basic_json/operator_spaceship.md @@ -6,7 +6,7 @@ class basic_json { std::partial_ordering operator<=>(const_reference rhs) const noexcept; // (1) template - std::partial_ordering operator<=>(const ScalarType rhs) const noexcept; // (2) + std::partial_ordering operator<=>(const ScalarType rhs) const noexcept(/* see below */); // (2) }; ``` @@ -39,7 +39,12 @@ the `std::partial_ordering` of the 3-way comparison of `*this` and `rhs` ## Exception safety -No-throw guarantee: this function never throws exceptions. +1. No-throw guarantee: this function never throws exceptions. +2. No-throw guarantee if converting the scalar to a JSON value cannot throw, as for numbers, Booleans, and + `#!cpp nullptr`; the function is `#!cpp noexcept` exactly in that case. Otherwise, it throws what the conversion + throws, for example `std::bad_alloc` when converting a string, or + [`out_of_range.410`](../../home/exceptions.md#jsonexceptionout_of_range410) for an enum value not mapped by + [`NLOHMANN_JSON_SERIALIZE_ENUM_STRICT`](../macros/nlohmann_json_serialize_enum_strict.md). ## Complexity @@ -98,3 +103,4 @@ Linear. 1. Added in version 3.11.0. 2. Added in version 3.11.0. + Made conditionally `#!cpp noexcept` in version 3.13.0; before, a throwing conversion called `std::terminate`. diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index d0f779109..bb78b211d 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -4865,7 +4865,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/operator_eq/ template requires std::is_scalar_v - bool operator==(ScalarType rhs) const noexcept + bool operator==(ScalarType rhs) const noexcept(std::is_nothrow_constructible::value) { return *this == basic_json(rhs); } @@ -4888,7 +4888,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/operator_spaceship/ template requires std::is_scalar_v - std::partial_ordering operator<=>(ScalarType rhs) const noexcept // *NOPAD* + std::partial_ordering operator<=>(ScalarType rhs) const noexcept(std::is_nothrow_constructible::value) // *NOPAD* { return *this <=> basic_json(rhs); // *NOPAD* } @@ -4913,7 +4913,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/operator_le/ template requires std::is_scalar_v - bool operator<=(ScalarType rhs) const noexcept + bool operator<=(ScalarType rhs) const noexcept(std::is_nothrow_constructible::value) { return *this <= basic_json(rhs); } @@ -4934,7 +4934,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/operator_ge/ template requires std::is_scalar_v - bool operator>=(ScalarType rhs) const noexcept + bool operator>=(ScalarType rhs) const noexcept(std::is_nothrow_constructible::value) { return *this >= basic_json(rhs); } @@ -4959,7 +4959,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/operator_eq/ template::value, int>::type = 0> - friend bool operator==(const_reference lhs, ScalarType rhs) noexcept + friend bool operator==(const_reference lhs, ScalarType rhs) noexcept(std::is_nothrow_constructible::value) { return lhs == basic_json(rhs); } @@ -4968,7 +4968,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/operator_eq/ template::value, int>::type = 0> - friend bool operator==(ScalarType lhs, const_reference rhs) noexcept + friend bool operator==(ScalarType lhs, const_reference rhs) noexcept(std::is_nothrow_constructible::value) { return basic_json(lhs) == rhs; } @@ -4984,7 +4984,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/operator_ne/ template::value, int>::type = 0> - friend bool operator!=(const_reference lhs, ScalarType rhs) noexcept + friend bool operator!=(const_reference lhs, ScalarType rhs) noexcept(std::is_nothrow_constructible::value) { return lhs != basic_json(rhs); } @@ -4993,7 +4993,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/operator_ne/ template::value, int>::type = 0> - friend bool operator!=(ScalarType lhs, const_reference rhs) noexcept + friend bool operator!=(ScalarType lhs, const_reference rhs) noexcept(std::is_nothrow_constructible::value) { return basic_json(lhs) != rhs; } @@ -5013,7 +5013,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/operator_lt/ template::value, int>::type = 0> - friend bool operator<(const_reference lhs, ScalarType rhs) noexcept + friend bool operator<(const_reference lhs, ScalarType rhs) noexcept(std::is_nothrow_constructible::value) { return lhs < basic_json(rhs); } @@ -5022,7 +5022,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/operator_lt/ template::value, int>::type = 0> - friend bool operator<(ScalarType lhs, const_reference rhs) noexcept + friend bool operator<(ScalarType lhs, const_reference rhs) noexcept(std::is_nothrow_constructible::value) { return basic_json(lhs) < rhs; } @@ -5042,7 +5042,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/operator_le/ template::value, int>::type = 0> - friend bool operator<=(const_reference lhs, ScalarType rhs) noexcept + friend bool operator<=(const_reference lhs, ScalarType rhs) noexcept(std::is_nothrow_constructible::value) { return lhs <= basic_json(rhs); } @@ -5051,7 +5051,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/operator_le/ template::value, int>::type = 0> - friend bool operator<=(ScalarType lhs, const_reference rhs) noexcept + friend bool operator<=(ScalarType lhs, const_reference rhs) noexcept(std::is_nothrow_constructible::value) { return basic_json(lhs) <= rhs; } @@ -5072,7 +5072,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/operator_gt/ template::value, int>::type = 0> - friend bool operator>(const_reference lhs, ScalarType rhs) noexcept + friend bool operator>(const_reference lhs, ScalarType rhs) noexcept(std::is_nothrow_constructible::value) { return lhs > basic_json(rhs); } @@ -5081,7 +5081,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/operator_gt/ template::value, int>::type = 0> - friend bool operator>(ScalarType lhs, const_reference rhs) noexcept + friend bool operator>(ScalarType lhs, const_reference rhs) noexcept(std::is_nothrow_constructible::value) { return basic_json(lhs) > rhs; } @@ -5101,7 +5101,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/operator_ge/ template::value, int>::type = 0> - friend bool operator>=(const_reference lhs, ScalarType rhs) noexcept + friend bool operator>=(const_reference lhs, ScalarType rhs) noexcept(std::is_nothrow_constructible::value) { return lhs >= basic_json(rhs); } @@ -5110,7 +5110,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/operator_ge/ template::value, int>::type = 0> - friend bool operator>=(ScalarType lhs, const_reference rhs) noexcept + friend bool operator>=(ScalarType lhs, const_reference rhs) noexcept(std::is_nothrow_constructible::value) { return basic_json(lhs) >= rhs; } diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 5a6f5afd7..c4f3313ae 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -31564,7 +31564,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/operator_eq/ template requires std::is_scalar_v - bool operator==(ScalarType rhs) const noexcept + bool operator==(ScalarType rhs) const noexcept(std::is_nothrow_constructible::value) { return *this == basic_json(rhs); } @@ -31587,7 +31587,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/operator_spaceship/ template requires std::is_scalar_v - std::partial_ordering operator<=>(ScalarType rhs) const noexcept // *NOPAD* + std::partial_ordering operator<=>(ScalarType rhs) const noexcept(std::is_nothrow_constructible::value) // *NOPAD* { return *this <=> basic_json(rhs); // *NOPAD* } @@ -31612,7 +31612,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/operator_le/ template requires std::is_scalar_v - bool operator<=(ScalarType rhs) const noexcept + bool operator<=(ScalarType rhs) const noexcept(std::is_nothrow_constructible::value) { return *this <= basic_json(rhs); } @@ -31633,7 +31633,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/operator_ge/ template requires std::is_scalar_v - bool operator>=(ScalarType rhs) const noexcept + bool operator>=(ScalarType rhs) const noexcept(std::is_nothrow_constructible::value) { return *this >= basic_json(rhs); } @@ -31658,7 +31658,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/operator_eq/ template::value, int>::type = 0> - friend bool operator==(const_reference lhs, ScalarType rhs) noexcept + friend bool operator==(const_reference lhs, ScalarType rhs) noexcept(std::is_nothrow_constructible::value) { return lhs == basic_json(rhs); } @@ -31667,7 +31667,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/operator_eq/ template::value, int>::type = 0> - friend bool operator==(ScalarType lhs, const_reference rhs) noexcept + friend bool operator==(ScalarType lhs, const_reference rhs) noexcept(std::is_nothrow_constructible::value) { return basic_json(lhs) == rhs; } @@ -31683,7 +31683,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/operator_ne/ template::value, int>::type = 0> - friend bool operator!=(const_reference lhs, ScalarType rhs) noexcept + friend bool operator!=(const_reference lhs, ScalarType rhs) noexcept(std::is_nothrow_constructible::value) { return lhs != basic_json(rhs); } @@ -31692,7 +31692,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/operator_ne/ template::value, int>::type = 0> - friend bool operator!=(ScalarType lhs, const_reference rhs) noexcept + friend bool operator!=(ScalarType lhs, const_reference rhs) noexcept(std::is_nothrow_constructible::value) { return basic_json(lhs) != rhs; } @@ -31712,7 +31712,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/operator_lt/ template::value, int>::type = 0> - friend bool operator<(const_reference lhs, ScalarType rhs) noexcept + friend bool operator<(const_reference lhs, ScalarType rhs) noexcept(std::is_nothrow_constructible::value) { return lhs < basic_json(rhs); } @@ -31721,7 +31721,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/operator_lt/ template::value, int>::type = 0> - friend bool operator<(ScalarType lhs, const_reference rhs) noexcept + friend bool operator<(ScalarType lhs, const_reference rhs) noexcept(std::is_nothrow_constructible::value) { return basic_json(lhs) < rhs; } @@ -31741,7 +31741,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/operator_le/ template::value, int>::type = 0> - friend bool operator<=(const_reference lhs, ScalarType rhs) noexcept + friend bool operator<=(const_reference lhs, ScalarType rhs) noexcept(std::is_nothrow_constructible::value) { return lhs <= basic_json(rhs); } @@ -31750,7 +31750,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/operator_le/ template::value, int>::type = 0> - friend bool operator<=(ScalarType lhs, const_reference rhs) noexcept + friend bool operator<=(ScalarType lhs, const_reference rhs) noexcept(std::is_nothrow_constructible::value) { return basic_json(lhs) <= rhs; } @@ -31771,7 +31771,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/operator_gt/ template::value, int>::type = 0> - friend bool operator>(const_reference lhs, ScalarType rhs) noexcept + friend bool operator>(const_reference lhs, ScalarType rhs) noexcept(std::is_nothrow_constructible::value) { return lhs > basic_json(rhs); } @@ -31780,7 +31780,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/operator_gt/ template::value, int>::type = 0> - friend bool operator>(ScalarType lhs, const_reference rhs) noexcept + friend bool operator>(ScalarType lhs, const_reference rhs) noexcept(std::is_nothrow_constructible::value) { return basic_json(lhs) > rhs; } @@ -31800,7 +31800,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/operator_ge/ template::value, int>::type = 0> - friend bool operator>=(const_reference lhs, ScalarType rhs) noexcept + friend bool operator>=(const_reference lhs, ScalarType rhs) noexcept(std::is_nothrow_constructible::value) { return lhs >= basic_json(rhs); } @@ -31809,7 +31809,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/operator_ge/ template::value, int>::type = 0> - friend bool operator>=(ScalarType lhs, const_reference rhs) noexcept + friend bool operator>=(ScalarType lhs, const_reference rhs) noexcept(std::is_nothrow_constructible::value) { return basic_json(lhs) >= rhs; } diff --git a/tests/src/unit-conversions.cpp b/tests/src/unit-conversions.cpp index ff6e5e5c7..df8685529 100644 --- a/tests/src/unit-conversions.cpp +++ b/tests/src/unit-conversions.cpp @@ -1764,6 +1764,12 @@ TEST_CASE("Strict JSON to enum mapping") CHECK(json("herz").get() == strict_cards::herz); CHECK(json("karo").get() == strict_cards::karo); + // comparison of enum and json + CHECK(strict_cards::kreuz == json("kreuz")); + CHECK(strict_cards::pik == json("pik")); + CHECK(strict_cards::herz == json("herz")); + CHECK(strict_cards::karo == json("karo")); + // invalid json -> exception thrown json _; CHECK_THROWS_WITH_AS(_ = json("what?").get(), "[json.exception.out_of_range.410] enum value out of range for strict_cards: \"what?\"", json::out_of_range&); @@ -1771,6 +1777,12 @@ TEST_CASE("Strict JSON to enum mapping") // conversion of unmapped enum -> exception thrown CHECK_THROWS_WITH_AS(json(strict_cards::andere), "[json.exception.out_of_range.410] enum value out of range for strict_cards", json::out_of_range&); + // comparing an unmapped enum with json throws the same exception + // (the scalar comparison operators used to be noexcept, so this + // called std::terminate) + CHECK_THROWS_WITH_AS(static_cast(strict_cards::andere == json("andere")), "[json.exception.out_of_range.410] enum value out of range for strict_cards", json::out_of_range&); + CHECK_THROWS_WITH_AS(static_cast(json("andere") != strict_cards::andere), "[json.exception.out_of_range.410] enum value out of range for strict_cards", json::out_of_range&); + // invalid UTF-8 -> out_of_range.410, not the type_error.316 thrown while building the // message (regression test for #5667); such strings can reach get() unvalidated, // e.g. from from_cbor()/from_msgpack() (#5529) @@ -1792,12 +1804,21 @@ TEST_CASE("Strict JSON to enum mapping") CHECK(json("completed").get() == STRICT_TS_COMPLETED); CHECK(json().get() == STRICT_TS_INVALID); + // comparison of enum and json + CHECK(STRICT_TS_STOPPED == json("stopped")); + CHECK(STRICT_TS_RUNNING == json("running")); + CHECK(STRICT_TS_COMPLETED == json("completed")); + CHECK(STRICT_TS_INVALID == json()); + // invalid json -> exception thrown json _; CHECK_THROWS_WITH_AS(_ = json("what?").get(), "[json.exception.out_of_range.410] enum value out of range for StrictTaskState: \"what?\"", json::out_of_range&); // conversion of unmapped enum -> exception thrown CHECK_THROWS_WITH_AS(json(STRICT_TS_OTHER), "[json.exception.out_of_range.410] enum value out of range for StrictTaskState", json::out_of_range&); + + // comparing an unmapped enum with json throws the same exception + CHECK_THROWS_WITH_AS(static_cast(STRICT_TS_OTHER < json("x")), "[json.exception.out_of_range.410] enum value out of range for StrictTaskState", json::out_of_range&); } } diff --git a/tests/src/unit-noexcept.cpp b/tests/src/unit-noexcept.cpp index 637915f24..85361df24 100644 --- a/tests/src/unit-noexcept.cpp +++ b/tests/src/unit-noexcept.cpp @@ -54,6 +54,15 @@ static_assert(noexcept(json(pod {})), ""); static_assert(noexcept(std::declval().get()), ""); static_assert(!noexcept(std::declval().get()), ""); static_assert(noexcept(json(pod{})), ""); + +// comparing with a scalar is noexcept exactly when converting the scalar is +static_assert(noexcept(std::declval() == 1), ""); +static_assert(noexcept(1 != std::declval()), ""); +static_assert(noexcept(std::declval() < 2.5), ""); +static_assert(noexcept(nullptr == std::declval()), ""); +static_assert(!noexcept(std::declval() == "foo"), ""); +static_assert(!noexcept("foo" >= std::declval()), ""); +static_assert(noexcept(std::declval() == std::declval()), ""); } // namespace TEST_CASE("noexcept") From 78ddd95794881a2830fa47972f8b6bcfba842c13 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 4 Oct 2026 11:46:13 +0200 Subject: [PATCH 02/22] Document GCC < 11 incomplete-type error with optional members (#3669) (#5596) * Document GCC < 11 incomplete-type error with optional members (#3669) With GCC 10 and older in C++11/14 mode, a free to_json() for a type holding an optional (Dummy constructible from json) fails with "invalid use of incomplete type detector<...to_json_function...>". ADL for Dummy finds the unrelated to_json and closes an instantiation cycle through optional's converting constructor. The same error reproduces without the library, so it can't be fixed here. Add a FAQ entry explaining the cause and the hidden-friend workaround, recommend hidden friends in the arbitrary types docs, and add a regression test that keeps the workaround compiling on GCC 7-10. Signed-off-by: Niels Lohmann * Use the #3669 fixture's optional member in to_json Issue3669Holder::d is never read, so clang's -Weverything -Werror build (ci_test_clang) fails with -Wunused-private-field. Reference the member in the hidden-friend to_json; this does not affect the instantiation cycle the fixture reproduces. Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann --- docs/mkdocs/docs/features/arbitrary_types.md | 1 + docs/mkdocs/docs/home/faq.md | 45 ++++++++++++++++ tests/src/unit-regression2.cpp | 54 ++++++++++++++++++++ 3 files changed, 100 insertions(+) diff --git a/docs/mkdocs/docs/features/arbitrary_types.md b/docs/mkdocs/docs/features/arbitrary_types.md index f7f3326ce..53f9cbd07 100644 --- a/docs/mkdocs/docs/features/arbitrary_types.md +++ b/docs/mkdocs/docs/features/arbitrary_types.md @@ -79,6 +79,7 @@ Some important things: * When using `get()`, `your_type` **MUST** be [DefaultConstructible](https://en.cppreference.com/w/cpp/named_req/DefaultConstructible). (There is a way to bypass this requirement described later.) * In function `from_json`, use function [`at()`](../api/basic_json/at.md) to access the object values rather than `operator[]`. In case a key does not exist, `at` throws an exception that you can handle, whereas `operator[]` exhibits undefined behavior. * You do not need to add serializers or deserializers for STL types like `std::vector`: the library already implements these. +* If you control the type, consider defining `to_json`/`from_json` as `friend` functions inside the class ("hidden friends"). Argument-dependent lookup then only finds them for your type, which also avoids a [GCC < 11 compilation error](../home/faq.md#incomplete-detector-type-with-gcc-11). ??? example "Example: serialize a `person` to JSON with `to_json`" diff --git a/docs/mkdocs/docs/home/faq.md b/docs/mkdocs/docs/home/faq.md index ca113004c..852fd8ffd 100644 --- a/docs/mkdocs/docs/home/faq.md +++ b/docs/mkdocs/docs/home/faq.md @@ -305,6 +305,51 @@ Only very old NDKs (before r18), which defaulted to GCC and `gnustl`, lacked C++ `std::to_string`. If you run into this, update to a current NDK. +### Incomplete `detector` type with GCC < 11 + +!!! question + + Why does GCC 10 or older fail with `invalid use of incomplete type 'struct nlohmann::detail::detector<..., to_json_function, ...>'` for a type that holds an `optional` member? + +This happens with GCC 10 and older in C++11/C++14 mode when all of these hold: + +- a class `Holder` has an `optional` member (e.g., `boost::optional`), +- `Dummy` has a constructor taking a `json` value, and +- `to_json` for `Holder` is a free function in the namespace of `Dummy`. + +```cpp +class Dummy { + public: + explicit Dummy(const nlohmann::json& j); +}; + +class Holder { + boost::optional d; +}; + +void to_json(nlohmann::json& j, const Holder& h); // triggers the error +``` + +To decide whether `Dummy` is copyable, the compiler checks whether a `Dummy` can be converted to `json`. That check +looks up `to_json` via argument-dependent lookup, finds the unrelated `to_json` for `Holder`, and eventually asks again +whether `Dummy` is copyable. GCC before version 11 turns this cycle into a hard error; GCC 11 and later, Clang, and +C++17 mode compile the code. The same error shows up without this library whenever a constrained converting constructor +is involved, so the library can't avoid it. + +To work around this, define `to_json` (and `from_json`) as a *hidden friend* inside the class. That way, +argument-dependent lookup only finds it for `Holder`: + +```cpp +class Holder { + boost::optional d; + + friend void to_json(nlohmann::json& j, const Holder& h) { /* ... */ } +}; +``` + +The [`NLOHMANN_DEFINE_TYPE_INTRUSIVE`](../api/macros/nlohmann_define_type_intrusive.md) macros define hidden friends as +well. See [#3669](https://github.com/nlohmann/json/issues/3669) for details. + ### Missing STL function !!! question "Questions" diff --git a/tests/src/unit-regression2.cpp b/tests/src/unit-regression2.cpp index 281c160af..1ea5ac595 100644 --- a/tests/src/unit-regression2.cpp +++ b/tests/src/unit-regression2.cpp @@ -233,6 +233,52 @@ class my_allocator : public std::allocator }; }; +///////////////////////////////////////////////////////////////////// +// for #3669 +///////////////////////////////////////////////////////////////////// + +// mimics boost::optional's converting constructor, whose SFINAE check asks +// whether T is constructible from const U& +template +struct issue3669_is_constructible +{ + template()))> + static char test(int); + template + static long test(...); + static constexpr bool value = sizeof(test(0)) == 1; +}; + +template +class issue3669_optional +{ + public: + issue3669_optional() = default; + template + issue3669_optional(const issue3669_optional& /*unused*/, // NOLINT(google-explicit-constructor,hicpp-explicit-conversions) + typename std::enable_if::value, bool>::type /*unused*/ = true) {} +}; + +class Issue3669Dummy +{ + public: + explicit Issue3669Dummy(const json& /*unused*/) {} +}; + +class Issue3669Holder +{ + issue3669_optional d{}; + + // GCC < 11 (C++11/14) rejects a free to_json(json&, const Issue3669Holder&) + // here, because ADL for Issue3669Dummy finds it and closes an instantiation + // cycle; a hidden friend is only visible to ADL for Issue3669Holder + friend void to_json(json& j, const Issue3669Holder& h) + { + static_cast(h.d); // silence -Wunused-private-field + j = "holder"; + } +}; + TEST_CASE("regression tests 2") { SECTION("issue #1001 - Fix memory leak during parser callback") @@ -762,6 +808,14 @@ TEST_CASE("regression tests 2") CHECK(j == k); } + SECTION("issue #3669 - invalid use of incomplete type with optional member and to_json") + { + const Issue3669Holder h{}; + const Issue3669Holder h2(h); // NOLINT(performance-unnecessary-copy-initialization) + const json j = h2; + CHECK(j == "holder"); + } + } TEST_CASE("regression test - parser callback must not lose a duplicate key's prior value") From d11e89471bdf0223b4a97fbfb25d95e36c8cd91a Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 4 Oct 2026 11:46:18 +0200 Subject: [PATCH 03/22] Assert on missing array indices in const operator[] and document the JSON pointer case (#5606) The const operator[] overloads are unchecked by design, and a missing key or index is undefined behavior. The key overload guards this with a runtime assertion, but the index overload did not, although the element access documentation says an assertion fires in both cases. The const JSON pointer overload inherits both through json_pointer::get_unchecked(), so a pointer to a missing array index read out of bounds even in debug builds, and its documentation promised out_of_range.404 for any pointer that cannot be resolved. Add JSON_ASSERT(idx < size()) to const operator[](size_type), which also covers the index leg of the const JSON pointer overload. Document the undefined behavior for the const JSON pointer overload in operator[].md and in the runtime assertions page. Release builds are unchanged. Signed-off-by: Niels Lohmann --- docs/mkdocs/docs/api/basic_json/operator[].md | 14 +++++-- docs/mkdocs/docs/features/assertions.md | 38 +++++++++++++++---- include/nlohmann/detail/json_pointer.hpp | 10 ++++- include/nlohmann/json.hpp | 1 + single_include/nlohmann/json.hpp | 11 +++++- 5 files changed, 60 insertions(+), 14 deletions(-) diff --git a/docs/mkdocs/docs/api/basic_json/operator[].md b/docs/mkdocs/docs/api/basic_json/operator[].md index 23926d5f6..74996d629 100644 --- a/docs/mkdocs/docs/api/basic_json/operator[].md +++ b/docs/mkdocs/docs/api/basic_json/operator[].md @@ -89,6 +89,9 @@ Strong exception safety: if an exception occurs, the original value stays intact - Throws [`out_of_range.410`](../../home/exceptions.md#jsonexceptionout_of_range410) if an array index in the passed JSON pointer `ptr` exceeds the range of `size_type` (e.g., on 32-bit platforms). + For the **const** version, an object key or array index in `ptr` that does not exist is not reported by an + exception, but is undefined behavior (see the notes below). Use [`at`](at.md) for checked access. + ## Complexity 1. Constant if `idx` is in the range of the array. Otherwise, linear in `idx - size()`. @@ -103,9 +106,12 @@ Strong exception safety: if an exception occurs, the original value stays intact The following cases apply to the **const** overloads; the non-const overloads instead insert the missing element (see the notes below). - 1. If the element at index `idx` does not exist, the behavior is undefined. + 1. If the element at index `idx` does not exist, the behavior is undefined and is **guarded by a + [runtime assertion](../../features/assertions.md)**! 2. If the element with key `key` does not exist, the behavior is undefined and is **guarded by a [runtime assertion](../../features/assertions.md)**! + 3. If the JSON pointer `ptr` refers to an object key or an array index that does not exist, the behavior is + undefined and is **guarded by a [runtime assertion](../../features/assertions.md)**! 1. The non-const version may add values: If `idx` is beyond the range of the array (i.e., `idx >= size()`), then the array is silently filled up with `#!json null` values to make `idx` a valid reference to the last stored element. In @@ -273,9 +279,11 @@ Strong exception safety: if an exception occurs, the original value stays intact ## Version history 1. Added in version 1.0.0. Fixed in version 3.13.0 to throw `#!cpp std::length_error` instead of emptying the array and - accessing it out of bounds when `idx` equals the maximum value of `size_type`. + accessing it out of bounds when `idx` equals the maximum value of `size_type`. A missing index in the const version + is guarded by a runtime assertion since version 3.13.0. 2. Added in version 1.0.0. Added overloads for `T* key` in version 1.1.0. Removed overloads for `T* key` (replaced by 3) in version 3.11.0. 3. Added in version 3.11.0. Fixed in version 3.13.0 to consistently accept `std::string_view`-convertible keys, as already supported by [`at`](at.md), [`value`](value.md), [`find`](find.md), and other lookup functions. -4. Added in version 2.0.0. +4. Added in version 2.0.0. A missing array index in the const version is guarded by a runtime assertion since + version 3.13.0. diff --git a/docs/mkdocs/docs/features/assertions.md b/docs/mkdocs/docs/features/assertions.md index e0b850115..dadb0bd87 100644 --- a/docs/mkdocs/docs/features/assertions.md +++ b/docs/mkdocs/docs/features/assertions.md @@ -16,14 +16,15 @@ before including the `json.hpp` header. ## Function with runtime assertions -### Unchecked object access to a const value +### Unchecked access to a const value -Function [`operator[]`](../api/basic_json/operator%5B%5D.md) implements unchecked access for objects. Whereas a missing -key is added in the case of non-const objects, accessing a const object with a missing key is undefined behavior (think -of a dereferenced null pointer) and yields a runtime assertion. +Function [`operator[]`](../api/basic_json/operator%5B%5D.md) implements unchecked access for arrays and objects. Whereas +a missing element is added in the case of non-const values, accessing a const value with a missing object key or an +invalid array index is undefined behavior (think of a dereferenced null pointer) and yields a runtime assertion. This +also applies to a [JSON pointer](json_pointer.md) that refers to a missing key or an invalid index. -If you are not sure whether an element in an object exists, use checked access with the -[`at` function](../api/basic_json/at.md) or call the [`contains` function](../api/basic_json/contains.md) before. +If you are not sure whether an element exists, use checked access with the [`at` function](../api/basic_json/at.md) +or call the [`contains` function](../api/basic_json/contains.md) before. See also the documentation on [element access](element_access/index.md). @@ -46,7 +47,30 @@ See also the documentation on [element access](element_access/index.md). Output: ``` - Assertion failed: (m_value.object->find(key) != m_value.object->end()), function operator[], file json.hpp, line 2144. + Assertion failed: (it != m_data.m_value.object->end()), function operator[], file json.hpp, line 28795. + ``` + +??? example "Example 2: Invalid array index in a JSON pointer" + + The following code will trigger an assertion at runtime: + + ```cpp + #include + + using json = nlohmann::json; + using namespace nlohmann::literals; + + int main() + { + const json j = {{"array", {1, 2, 3}}}; + auto v = j["/array/5"_json_pointer]; + } + ``` + + Output: + + ``` + Assertion failed: (idx < m_data.m_value.array->size()), function operator[], file json.hpp, line 28758. ``` ### Constructing from an uninitialized iterator range diff --git a/include/nlohmann/detail/json_pointer.hpp b/include/nlohmann/detail/json_pointer.hpp index cf3720a25..e34acb6d3 100644 --- a/include/nlohmann/detail/json_pointer.hpp +++ b/include/nlohmann/detail/json_pointer.hpp @@ -536,6 +536,10 @@ class json_pointer @return const reference to the JSON value pointed to by the JSON pointer + @pre Every object key and array index the pointer refers to exists. + Like the const operator[] for keys and indices, a missing one is + undefined behavior, guarded by a runtime assertion. + @throw parse_error.106 if an array index begins with '0' @throw parse_error.109 if an array index was not a number @throw out_of_range.402 if the array index '-' is used @@ -550,7 +554,8 @@ class json_pointer { case detail::value_t::object: { - // use unchecked object access + // use unchecked object access; the const operator[] + // asserts that the key exists ptr = &ptr->operator[](reference_token); break; } @@ -563,7 +568,8 @@ class json_pointer JSON_THROW(detail::out_of_range::create(402, detail::concat("array index '-' (", std::to_string(ptr->m_data.m_value.array->size()), ") is out of range"), ptr)); } - // use unchecked array access + // use unchecked array access; the const operator[] + // asserts that the index exists ptr = &ptr->operator[](array_index(reference_token)); break; } diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index bb78b211d..1ff12c7f3 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -3065,6 +3065,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // const operator[] only works for arrays if (JSON_HEDLEY_LIKELY(is_array())) { + JSON_ASSERT(idx < m_data.m_value.array->size()); return m_data.m_value.array->operator[](idx); } diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index c4f3313ae..9e27f62f0 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -20105,6 +20105,10 @@ class json_pointer @return const reference to the JSON value pointed to by the JSON pointer + @pre Every object key and array index the pointer refers to exists. + Like the const operator[] for keys and indices, a missing one is + undefined behavior, guarded by a runtime assertion. + @throw parse_error.106 if an array index begins with '0' @throw parse_error.109 if an array index was not a number @throw out_of_range.402 if the array index '-' is used @@ -20119,7 +20123,8 @@ class json_pointer { case detail::value_t::object: { - // use unchecked object access + // use unchecked object access; the const operator[] + // asserts that the key exists ptr = &ptr->operator[](reference_token); break; } @@ -20132,7 +20137,8 @@ class json_pointer JSON_THROW(detail::out_of_range::create(402, detail::concat("array index '-' (", std::to_string(ptr->m_data.m_value.array->size()), ") is out of range"), ptr)); } - // use unchecked array access + // use unchecked array access; the const operator[] + // asserts that the index exists ptr = &ptr->operator[](array_index(reference_token)); break; } @@ -29764,6 +29770,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // const operator[] only works for arrays if (JSON_HEDLEY_LIKELY(is_array())) { + JSON_ASSERT(idx < m_data.m_value.array->size()); return m_data.m_value.array->operator[](idx); } From df27cc3d4cdf9ecb9337524c2af7752967ebf4ca Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 4 Oct 2026 11:46:21 +0200 Subject: [PATCH 04/22] Add scalar-on-left overloads for legacy discarded comparisons in C++20 (#5682) With JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON=1 and C++20, a scalar on the left-hand side of <= or >= (e.g., `1 <= discarded`) yielded false instead of the documented true. The C++20 legacy block only had member operators, which are only candidates when the basic_json is the left operand; for a scalar on the left, overload resolution picked the candidate rewritten from operator<=>, which does not emulate the legacy behavior. The C++17 branch already has scalar-on-the-left friend overloads for <= and >=; add the equivalent pair to the C++20 legacy block. Added a regression test to tests/src/unit-comparison.cpp covering all four operand orders for both operators. Fixes #5665. Signed-off-by: Niels Lohmann --- ...n_use_legacy_discarded_value_comparison.md | 3 +++ include/nlohmann/json.hpp | 21 ++++++++++++++++++ single_include/nlohmann/json.hpp | 21 ++++++++++++++++++ tests/src/unit-comparison.cpp | 22 +++++++++++++++++++ 4 files changed, 67 insertions(+) diff --git a/docs/mkdocs/docs/api/macros/json_use_legacy_discarded_value_comparison.md b/docs/mkdocs/docs/api/macros/json_use_legacy_discarded_value_comparison.md index b6efd8dbd..e61ce1a3e 100644 --- a/docs/mkdocs/docs/api/macros/json_use_legacy_discarded_value_comparison.md +++ b/docs/mkdocs/docs/api/macros/json_use_legacy_discarded_value_comparison.md @@ -81,3 +81,6 @@ When the macro is not defined, the library will define it to its default value. ## Version history - Added in version 3.11.0. +- Fixed in version 3.13.0 so `<=` and `>=` also emulate the legacy behavior in C++20 when the JSON value is the + right-hand operand of a scalar comparison; before, only the 3-way-comparison-rewritten candidate was found, which + yielded `#!cpp false` instead of `#!cpp true`. diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 1ff12c7f3..b802b4b59 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -4939,6 +4939,27 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec { return *this >= basic_json(rhs); } + + // a scalar on the left-hand side would otherwise select the candidate + // rewritten from operator<=>, which does not emulate the legacy behavior + + /// @brief comparison: less than or equal + /// @sa https://json.nlohmann.me/api/basic_json/operator_le/ + template + requires std::is_scalar_v + friend bool operator<=(ScalarType lhs, const_reference rhs) noexcept + { + return basic_json(lhs) <= rhs; + } + + /// @brief comparison: greater than or equal + /// @sa https://json.nlohmann.me/api/basic_json/operator_ge/ + template + requires std::is_scalar_v + friend bool operator>=(ScalarType lhs, const_reference rhs) noexcept + { + return basic_json(lhs) >= rhs; + } #endif #else /// @brief comparison: equal diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 9e27f62f0..2adc9978f 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -31644,6 +31644,27 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec { return *this >= basic_json(rhs); } + + // a scalar on the left-hand side would otherwise select the candidate + // rewritten from operator<=>, which does not emulate the legacy behavior + + /// @brief comparison: less than or equal + /// @sa https://json.nlohmann.me/api/basic_json/operator_le/ + template + requires std::is_scalar_v + friend bool operator<=(ScalarType lhs, const_reference rhs) noexcept + { + return basic_json(lhs) <= rhs; + } + + /// @brief comparison: greater than or equal + /// @sa https://json.nlohmann.me/api/basic_json/operator_ge/ + template + requires std::is_scalar_v + friend bool operator>=(ScalarType lhs, const_reference rhs) noexcept + { + return basic_json(lhs) >= rhs; + } #endif #else /// @brief comparison: equal diff --git a/tests/src/unit-comparison.cpp b/tests/src/unit-comparison.cpp index febfd9b42..a65d5c1b3 100644 --- a/tests/src/unit-comparison.cpp +++ b/tests/src/unit-comparison.cpp @@ -750,6 +750,28 @@ TEST_CASE("regression #3868 - heterogeneous comparisons compile under C++20 (P24 CHECK_FALSE(j != i); } } + +#if JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON +TEST_CASE("regression #5665 - scalar <= discarded and scalar >= discarded in C++20 legacy mode") +{ + // Issue #5665: with a scalar on the left-hand side, <= and >= only had the + // candidate rewritten from operator<=>, which does not emulate the legacy + // discarded-value behavior. Check that scalar-on-the-left now matches the + // other three operand orders. + const json discarded(json::value_t::discarded); + const json one = 1; + + CHECK(discarded <= 1); + CHECK(discarded >= 1); + CHECK(one <= discarded); + CHECK(one >= discarded); + CHECK(1 <= discarded); + CHECK(1 >= discarded); + CHECK(1.5 <= discarded); + CHECK(1.5 >= discarded); +} +#endif + #endif namespace From 6f2048cd5d609c055742d91539457567a463cb4d Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 4 Oct 2026 11:46:25 +0200 Subject: [PATCH 05/22] Accept lvalues in ordered_map::emplace's value parameter (#5685) * Accept lvalues in ordered_map::emplace's value parameter ordered_map::emplace(key, value) took the mapped value only by T&&, an rvalue reference rather than a forwarding reference, so ordered_json::emplace("a", value) failed to compile whenever value was an lvalue or a const lvalue, even though the same call compiles for json (whose object_t is std::map, with a variadic emplace). Turn the value parameter into a separately-deduced forwarding reference, constrained with std::is_constructible so the overloads still only accept something convertible to the mapped type. std::map-compatible semantics are unchanged: emplace still does nothing if the key already exists. Open PR #5609 also touches ordered_map.hpp (moving values on vector growth); this change only touches the two emplace() overloads and should not conflict. Fixes #5673. Signed-off-by: Niels Lohmann * Avoid astyle's padding in ordered_map::emplace's template headers Use detail::conjunction instead of && and drop the redundant V&& in detail::is_constructible, so astyle keeps the usual template formatting. Addresses review comment by @gregmarr. Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann --- docs/mkdocs/docs/api/basic_json/emplace.md | 2 + include/nlohmann/ordered_map.hpp | 15 +++-- single_include/nlohmann/json.hpp | 15 +++-- tests/src/unit-ordered_json.cpp | 41 ++++++++++++ tests/src/unit-ordered_map.cpp | 73 ++++++++++++++++++++++ 5 files changed, 134 insertions(+), 12 deletions(-) diff --git a/docs/mkdocs/docs/api/basic_json/emplace.md b/docs/mkdocs/docs/api/basic_json/emplace.md index 26044a597..09953d347 100644 --- a/docs/mkdocs/docs/api/basic_json/emplace.md +++ b/docs/mkdocs/docs/api/basic_json/emplace.md @@ -70,3 +70,5 @@ Logarithmic in the size of the container, O(log(`size()`)). ## Version history - Since version 2.0.8. +- Fixed in version 3.13.0: for [`ordered_json`](../ordered_json.md), the value could previously only be passed as an + rvalue; it can now also be passed as an lvalue or a `#!cpp const` lvalue, matching the behavior of `json`. diff --git a/include/nlohmann/ordered_map.hpp b/include/nlohmann/ordered_map.hpp index 656f24264..d1c247483 100644 --- a/include/nlohmann/ordered_map.hpp +++ b/include/nlohmann/ordered_map.hpp @@ -74,7 +74,9 @@ template , return *this; } - std::pair emplace(const key_type& key, T&& t) + template::value, int> = 0> + std::pair emplace(const key_type& key, V && t) { for (auto it = this->begin(); it != this->end(); ++it) { @@ -83,13 +85,14 @@ template , return {it, false}; } } - append(key, std::forward(t)); + append(key, std::forward(t)); return {std::prev(this->end()), true}; } - template::value, int> = 0> - std::pair emplace(KeyType && key, T && t) + template, + detail::is_constructible>::value, int> = 0> + std::pair emplace(KeyType && key, V && t) { for (auto it = this->begin(); it != this->end(); ++it) { @@ -98,7 +101,7 @@ template , return {it, false}; } } - append(std::forward(key), std::forward(t)); + append(std::forward(key), std::forward(t)); return {std::prev(this->end()), true}; } diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 2adc9978f..fe3547ab2 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -26401,7 +26401,9 @@ template , return *this; } - std::pair emplace(const key_type& key, T&& t) + template::value, int> = 0> + std::pair emplace(const key_type& key, V && t) { for (auto it = this->begin(); it != this->end(); ++it) { @@ -26410,13 +26412,14 @@ template , return {it, false}; } } - append(key, std::forward(t)); + append(key, std::forward(t)); return {std::prev(this->end()), true}; } - template::value, int> = 0> - std::pair emplace(KeyType && key, T && t) + template, + detail::is_constructible>::value, int> = 0> + std::pair emplace(KeyType && key, V && t) { for (auto it = this->begin(); it != this->end(); ++it) { @@ -26425,7 +26428,7 @@ template , return {it, false}; } } - append(std::forward(key), std::forward(t)); + append(std::forward(key), std::forward(t)); return {std::prev(this->end()), true}; } diff --git a/tests/src/unit-ordered_json.cpp b/tests/src/unit-ordered_json.cpp index 45fbf5493..135dedade 100644 --- a/tests/src/unit-ordered_json.cpp +++ b/tests/src/unit-ordered_json.cpp @@ -196,3 +196,44 @@ TEST_CASE("regression test - diff() must account for ordered_json member order") CHECK(a.patch(p) == b); } } + +TEST_CASE("regression test for issue #5673 - ordered_json::emplace with a non-rvalue value") +{ + SECTION("lvalue value") + { + ordered_json oj = ordered_json::object(); + ordered_json value = 1; + auto res = oj.emplace("a", value); + CHECK(res.second == true); + CHECK(oj.dump() == "{\"a\":1}"); + } + + SECTION("const lvalue value") + { + ordered_json oj = ordered_json::object(); + const ordered_json value = 1; + auto res = oj.emplace("a", value); + CHECK(res.second == true); + CHECK(oj.dump() == "{\"a\":1}"); + } + + SECTION("rvalue value") + { + ordered_json oj = ordered_json::object(); + auto res = oj.emplace("a", ordered_json(1)); + CHECK(res.second == true); + CHECK(oj.dump() == "{\"a\":1}"); + } + + SECTION("existing key is not overwritten (std::map-compatible semantics)") + { + ordered_json oj = ordered_json::object(); + ordered_json value = 1; + oj.emplace("a", value); + + ordered_json other_value = 2; + auto res = oj.emplace("a", other_value); + CHECK(res.second == false); + CHECK(oj.dump() == "{\"a\":1}"); + } +} diff --git a/tests/src/unit-ordered_map.cpp b/tests/src/unit-ordered_map.cpp index 98b6fa0d1..dce3f61a5 100644 --- a/tests/src/unit-ordered_map.cpp +++ b/tests/src/unit-ordered_map.cpp @@ -403,6 +403,79 @@ TEST_CASE("ordered_map") CHECK(om.size() == 4); } } + + SECTION("emplace") + { + // regression test for issue #5673: the mapped-value parameter must + // accept lvalues and const lvalues, not just rvalues + ordered_map om; + om["eins"] = "one"; + om["zwei"] = "two"; + om["drei"] = "three"; + + SECTION("with T&& (rvalue)") + { + auto res1 = om.emplace("eins", std::string("1")); + CHECK(res1.first == om.begin()); + CHECK(res1.second == false); + CHECK(om.size() == 3); + CHECK(om.at("eins") == "one"); // existing key is not overwritten + + auto res4 = om.emplace("vier", std::string("four")); + CHECK(res4.first == om.begin() + 3); + CHECK(res4.second == true); + CHECK(om.size() == 4); + CHECK(om.at("vier") == "four"); + } + + SECTION("with T& (lvalue)") + { + std::string one = "1"; + std::string four = "four"; + + auto res1 = om.emplace("eins", one); + CHECK(res1.first == om.begin()); + CHECK(res1.second == false); + CHECK(om.size() == 3); + CHECK(om.at("eins") == "one"); // existing key is not overwritten + + auto res4 = om.emplace("vier", four); + CHECK(res4.first == om.begin() + 3); + CHECK(res4.second == true); + CHECK(om.size() == 4); + CHECK(om.at("vier") == "four"); + CHECK(four == "four"); // source was copied, not moved from + } + + SECTION("with const T&") + { + const std::string one = "1"; + const std::string four = "four"; + + auto res1 = om.emplace("eins", one); + CHECK(res1.first == om.begin()); + CHECK(res1.second == false); + CHECK(om.size() == 3); + + auto res4 = om.emplace("vier", four); + CHECK(res4.first == om.begin() + 3); + CHECK(res4.second == true); + CHECK(om.size() == 4); + CHECK(om.at("vier") == "four"); + } + + SECTION("with key of key_type (non-template overload)") + { + const std::string key_vier{"vier"}; + std::string four = "four"; + + auto res4 = om.emplace(key_vier, four); + CHECK(res4.first == om.begin() + 3); + CHECK(res4.second == true); + CHECK(om.size() == 4); + CHECK(om.at("vier") == "four"); + } + } } TEST_CASE("ordered_map growth") From 0d01d6ae9074e167bcdca0bc6e592c02e900c692 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 4 Oct 2026 11:46:29 +0200 Subject: [PATCH 06/22] Classify leaves with operator<=> itself past the nesting bound (#5686) * Classify leaves with operator<=> itself past the nesting bound In C++20, an ordered comparison past the nesting bound classified a pair of leaves by asking == first and then order_leaves(), which calls < and > - both derived from <=>. For a pair of binary values with the same bytes but a different subtype, == reports them unequal, while <=> (through std::vector::operator<=>) reports them equivalent, so the pair ended the comparison as unordered instead of letting the next element decide - unlike an array or object within the bound, which compares such a pair with its own operator<=> and gets equivalent. So operator<=>, and the <, <=, >, >= derived from it, could give a different result for the same two values depending on how deeply the values were nested, or unordered at every depth with JSON_NO_THREAD_LOCAL defined. compare_leaves() now classifies such a pair in C++20 with operator<=> itself instead, matching how a value within the bound is compared; the equality-only and pre-C++20 ordered cases are unchanged. Which of the three runs is chosen by overloading on std::integral_constant, the same tag dispatch order_leaves() already uses, rather than a runtime "if (Ordered)" on a template parameter, which MSVC would flag as a constant condition (C4127). Added a regression test to unit-comparison.cpp that nests such a pair 0, 127, 128 and 200 levels deep (127 stays within the 128-level bound, 128 and 200 do not) and checks that operator<=> and operator< agree at every depth. Fixes #5654. Signed-off-by: Niels Lohmann * Drop the version history note for a bug that was never released The regression came from #5390, which is not in any release. Addresses review comment by @gregmarr. Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann --- include/nlohmann/json.hpp | 52 +++++++++++++++++++++++++++++++- single_include/nlohmann/json.hpp | 52 +++++++++++++++++++++++++++++++- tests/src/unit-comparison.cpp | 48 +++++++++++++++++++++++++++++ 3 files changed, 150 insertions(+), 2 deletions(-) diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index b802b4b59..bf3052748 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -1528,15 +1528,65 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec */ template static compare_result compare_leaves(const_reference lhs, const_reference rhs) noexcept + { + return compare_leaves(lhs, rhs, std::integral_constant {}); + } + + /// @brief compare two leaves that are only being checked for equality + static compare_result compare_leaves(const_reference lhs, const_reference rhs, std::false_type /*ordered*/) noexcept { if (lhs == rhs) { return compare_result::equal; } - return order_leaves(lhs, rhs, std::integral_constant {}); + return order_leaves(lhs, rhs, std::false_type {}); } +#if JSON_HAS_THREE_WAY_COMPARISON + /*! + @brief compare two leaves that are being ordered, for operator<=> + + Reached only from operator<=>, so the leaves must be classified exactly + as operator<=> classifies them - which is not the same as asking + == and then order_leaves(), the way the other overload does it. The two + disagree on a binary value: == also compares the subtype, but <=> compares + only the bytes, through std::vector::operator<=>. Using <=> + itself here keeps a leaf pair classified the same way regardless of how + deep it is nested - == first would again call operator<=> a level down + through order_leaves(), but call it after a mismatching == already ended + the comparison for a pair that <=> alone would still call equivalent. + */ + static compare_result compare_leaves(const_reference lhs, const_reference rhs, std::true_type /*ordered*/) noexcept + { + const std::partial_ordering order = lhs <=> rhs; // *NOPAD* + if (order == 0) + { + return compare_result::equal; + } + if (order < 0) + { + return compare_result::less; + } + if (order > 0) + { + return compare_result::greater; + } + return compare_result::unordered; + } +#else + /// @brief compare two leaves that are being ordered, for operator< + static compare_result compare_leaves(const_reference lhs, const_reference rhs, std::true_type /*ordered*/) noexcept + { + if (lhs == rhs) + { + return compare_result::equal; + } + + return order_leaves(lhs, rhs, std::true_type {}); + } +#endif + /*! @brief compare two object keys diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index fe3547ab2..3c1048003 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -28236,15 +28236,65 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec */ template static compare_result compare_leaves(const_reference lhs, const_reference rhs) noexcept + { + return compare_leaves(lhs, rhs, std::integral_constant {}); + } + + /// @brief compare two leaves that are only being checked for equality + static compare_result compare_leaves(const_reference lhs, const_reference rhs, std::false_type /*ordered*/) noexcept { if (lhs == rhs) { return compare_result::equal; } - return order_leaves(lhs, rhs, std::integral_constant {}); + return order_leaves(lhs, rhs, std::false_type {}); } +#if JSON_HAS_THREE_WAY_COMPARISON + /*! + @brief compare two leaves that are being ordered, for operator<=> + + Reached only from operator<=>, so the leaves must be classified exactly + as operator<=> classifies them - which is not the same as asking + == and then order_leaves(), the way the other overload does it. The two + disagree on a binary value: == also compares the subtype, but <=> compares + only the bytes, through std::vector::operator<=>. Using <=> + itself here keeps a leaf pair classified the same way regardless of how + deep it is nested - == first would again call operator<=> a level down + through order_leaves(), but call it after a mismatching == already ended + the comparison for a pair that <=> alone would still call equivalent. + */ + static compare_result compare_leaves(const_reference lhs, const_reference rhs, std::true_type /*ordered*/) noexcept + { + const std::partial_ordering order = lhs <=> rhs; // *NOPAD* + if (order == 0) + { + return compare_result::equal; + } + if (order < 0) + { + return compare_result::less; + } + if (order > 0) + { + return compare_result::greater; + } + return compare_result::unordered; + } +#else + /// @brief compare two leaves that are being ordered, for operator< + static compare_result compare_leaves(const_reference lhs, const_reference rhs, std::true_type /*ordered*/) noexcept + { + if (lhs == rhs) + { + return compare_result::equal; + } + + return order_leaves(lhs, rhs, std::true_type {}); + } +#endif + /*! @brief compare two object keys diff --git a/tests/src/unit-comparison.cpp b/tests/src/unit-comparison.cpp index a65d5c1b3..4b66f1d07 100644 --- a/tests/src/unit-comparison.cpp +++ b/tests/src/unit-comparison.cpp @@ -1020,3 +1020,51 @@ TEST_CASE("containers are compared element by element") } } } + +#if JSON_HAS_THREE_WAY_COMPARISON +// JSON_HAS_CPP_20 (do not remove; see note at top of file) +TEST_CASE("operator<=> of binary values with a different subtype does not depend on nesting depth") +{ + // #5654: std::vector::operator<=>, which the binary type's + // own operator<=> uses, ignores the subtype that operator== checks. So a + // pair of binary values with the same bytes but a different subtype is + // unequal, yet <=>-equivalent - the same inconsistency between == and <=> + // that a NaN has. Within the nesting bound, an array compares itself + // with std::vector's own operator<=>, which treats an equivalent pair as + // undecided and lets the next element decide, same as + // std::lexicographical_compare_three_way does. Past the bound, + // compare_iteratively() takes over and must classify the pair the + // same way, or the result of operator<=> - and of <, which C++20 derives + // from it - depends on how deeply the values are nested. + const json a = json::array({json::binary({1}, 1), 1}); + const json b = json::array({json::binary({1}, 2), 2}); + + // the root inconsistency: unequal, yet <=>-equivalent + CHECK_FALSE(a[0] == b[0]); + CHECK((a[0] <=> b[0]) == std::partial_ordering::equivalent); // *NOPAD* + + const auto deep = [](const json & j, const std::size_t depth) + { + json result = j; + for (std::size_t i = 0; i < depth; ++i) + { + result = json::array({std::move(result)}); + } + return result; + }; + + // 127 levels stay within nesting_depth_limit() (128); 128 and 200 do not, + // and must still agree with the levels that do + for (const std::size_t depth : std::vector {0, 127, 128, 200}) + { + CAPTURE(depth); + const json x = deep(a, depth); + const json y = deep(b, depth); + CHECK((x <=> y) == std::partial_ordering::less); // *NOPAD* + CHECK((y <=> x) == std::partial_ordering::greater); // *NOPAD* + CHECK(x < y); + CHECK(y > x); + CHECK_FALSE(y < x); + } +} +#endif From b730946432dd9b22ecd012515f50a191d96954fc Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 4 Oct 2026 11:46:32 +0200 Subject: [PATCH 07/22] Fix key types convertible to std::string_view breaking lookups (#5689) Since #4958, a key type implicitly convertible to std::string_view was accepted by is_usable_as_basic_json_key_type without checking that the object's comparator can actually compare object_t::key_type with that key type. The key was then forwarded unchanged to the underlying map, so const operator[], at, find, count, contains, erase and value failed to compile (a hard error inside ) for a key convertible only to std::string_view, and value() rejected such keys outright. For keys convertible to both std::string and std::string_view, the KeyType&& templates now won overload resolution over the object_t::key_type overloads and then failed the same way, a regression from 3.12.0. Only the non-const operator[] worked, because it uses emplace(), which constructs a std::string from the key explicitly. ordered_json was not affected, since ordered_map checks comparability itself. Add a trait, is_string_view_convertible_key_type, that recognizes a key type that is convertible to std::string_view but not directly comparable with the object's key type, provided std::string_view itself is comparable with it. at(), operator[], find(), count(), contains(), erase() and value() now route such keys through a new lookup_key() helper that converts them to std::string_view before they reach the object, matching how the object's transparent comparator already supports std::string_view lookups. Fixes #5663. Signed-off-by: Niels Lohmann --- docs/mkdocs/docs/api/basic_json/at.md | 4 +- docs/mkdocs/docs/api/basic_json/contains.md | 4 +- docs/mkdocs/docs/api/basic_json/count.md | 4 +- docs/mkdocs/docs/api/basic_json/erase.md | 4 +- docs/mkdocs/docs/api/basic_json/find.md | 4 +- docs/mkdocs/docs/api/basic_json/value.md | 4 +- include/nlohmann/detail/meta/type_traits.hpp | 28 ++++- include/nlohmann/json.hpp | 44 ++++++-- single_include/nlohmann/json.hpp | 72 +++++++++--- tests/src/unit-element_access2.cpp | 111 +++++++++++++++++++ 10 files changed, 243 insertions(+), 36 deletions(-) diff --git a/docs/mkdocs/docs/api/basic_json/at.md b/docs/mkdocs/docs/api/basic_json/at.md index 60daf38e3..18f108964 100644 --- a/docs/mkdocs/docs/api/basic_json/at.md +++ b/docs/mkdocs/docs/api/basic_json/at.md @@ -238,5 +238,7 @@ Strong exception safety: if an exception occurs, the original value stays intact 1. Added in version 1.0.0. 2. Added in version 1.0.0. -3. Added in version 3.11.0. +3. Added in version 3.11.0. Fixed in version 3.13.0 to consistently accept `std::string_view`-convertible keys, as + already supported by [`operator[]`](operator[].md), [`value`](value.md), [`find`](find.md), and other lookup + functions. 4. Added in version 2.0.0. diff --git a/docs/mkdocs/docs/api/basic_json/contains.md b/docs/mkdocs/docs/api/basic_json/contains.md index 73bcf6f0c..63ec87a1d 100644 --- a/docs/mkdocs/docs/api/basic_json/contains.md +++ b/docs/mkdocs/docs/api/basic_json/contains.md @@ -131,7 +131,9 @@ Logarithmic in the size of the JSON object. ## Version history 1. Added in version 3.11.0. -2. Added in version 3.6.0. Extended template `KeyType` to support comparable types in version 3.11.0. +2. Added in version 3.6.0. Extended template `KeyType` to support comparable types in version 3.11.0. Fixed in + version 3.13.0 to consistently accept `std::string_view`-convertible keys, as already supported by + [`operator[]`](operator[].md), [`at`](at.md), [`value`](value.md), and other lookup functions. 3. Added in version 3.7.0. 4. Deleted overloads for integral key types added in version 3.13.0 to reject such calls at compile time instead of causing undefined behavior at runtime. diff --git a/docs/mkdocs/docs/api/basic_json/count.md b/docs/mkdocs/docs/api/basic_json/count.md index bffc46534..14b707525 100644 --- a/docs/mkdocs/docs/api/basic_json/count.md +++ b/docs/mkdocs/docs/api/basic_json/count.md @@ -84,6 +84,8 @@ Logarithmic in the size of the JSON object. ## Version history 1. Added in version 3.11.0. -2. Added in version 1.0.0. Changed parameter `key` type to `KeyType&&` in version 3.11.0. +2. Added in version 1.0.0. Changed parameter `key` type to `KeyType&&` in version 3.11.0. Fixed in version 3.13.0 to + consistently accept `std::string_view`-convertible keys, as already supported by [`operator[]`](operator[].md), + [`at`](at.md), [`value`](value.md), and other lookup functions. 3. Deleted overload for integral key types added in version 3.13.0 to reject such calls at compile time instead of causing undefined behavior at runtime. diff --git a/docs/mkdocs/docs/api/basic_json/erase.md b/docs/mkdocs/docs/api/basic_json/erase.md index d1e6d6d22..47531fed8 100644 --- a/docs/mkdocs/docs/api/basic_json/erase.md +++ b/docs/mkdocs/docs/api/basic_json/erase.md @@ -213,5 +213,7 @@ Strong exception safety: if an exception occurs, the original value stays intact 1. Added in version 1.0.0. Added support for binary types in version 3.8.0. 2. Added in version 1.0.0. Added support for binary types in version 3.8.0. 3. Added in version 1.0.0. -4. Added in version 3.11.0. +4. Added in version 3.11.0. Fixed in version 3.13.0 to consistently accept `std::string_view`-convertible keys, as + already supported by [`operator[]`](operator[].md), [`at`](at.md), [`value`](value.md), and other lookup + functions. 5. Added in version 1.0.0. diff --git a/docs/mkdocs/docs/api/basic_json/find.md b/docs/mkdocs/docs/api/basic_json/find.md index bc746ee2f..59c2eab68 100644 --- a/docs/mkdocs/docs/api/basic_json/find.md +++ b/docs/mkdocs/docs/api/basic_json/find.md @@ -88,6 +88,8 @@ Logarithmic in the size of the JSON object. ## Version history 1. Added in version 3.11.0. -2. Added in version 1.0.0. Changed to support comparable types in version 3.11.0. +2. Added in version 1.0.0. Changed to support comparable types in version 3.11.0. Fixed in version 3.13.0 to + consistently accept `std::string_view`-convertible keys, as already supported by [`operator[]`](operator[].md), + [`at`](at.md), [`value`](value.md), and other lookup functions. 3. Deleted overloads for integral key types added in version 3.13.0 to reject such calls at compile time instead of causing undefined behavior at runtime. diff --git a/docs/mkdocs/docs/api/basic_json/value.md b/docs/mkdocs/docs/api/basic_json/value.md index 56f4bcc0f..80f5691dc 100644 --- a/docs/mkdocs/docs/api/basic_json/value.md +++ b/docs/mkdocs/docs/api/basic_json/value.md @@ -222,7 +222,9 @@ changes to any JSON value. 1. Added in version 1.0.0. Changed parameter `default_value` type from `const ValueType&` to `ValueType&&` in version 3.11.0. Deleted overload for integral key types added in version 3.13.0 to reject such calls at compile time instead of causing undefined behavior at runtime. -2. Added in version 3.11.0. Made `ValueType` the first template parameter in version 3.11.2. +2. Added in version 3.11.0. Made `ValueType` the first template parameter in version 3.11.2. Fixed in version 3.13.0 + to consistently accept `std::string_view`-convertible keys, as already supported by + [`operator[]`](operator[].md), [`at`](at.md), [`find`](find.md), and other lookup functions. 3. Added in version 2.0.2. Extended to work with arrays in version 3.13.0, including fixing an issue where resolving `ptr` through an array unexpectedly threw `out_of_range` instead of returning the resolved element (or `default_value`, as documented). diff --git a/include/nlohmann/detail/meta/type_traits.hpp b/include/nlohmann/detail/meta/type_traits.hpp index 96b70a774..2f837e046 100644 --- a/include/nlohmann/detail/meta/type_traits.hpp +++ b/include/nlohmann/detail/meta/type_traits.hpp @@ -760,6 +760,30 @@ using is_usable_as_key_type = typename std::conditional < std::true_type, std::false_type >::type; +#ifdef JSON_HAS_CPP_17 +// type trait to check if KeyType can only be used as an object key after +// converting it to std::string_view: it is convertible to std::string_view, the +// object's comparator cannot compare it with object_t::key_type directly, but +// can compare a std::string_view. JSON pointers and JSON iterators are ruled out +// first, so that the conversion checks are never instantiated for them (a JSON +// pointer's deprecated conversion to string_t would be named otherwise). +template < typename BasicJsonType, typename KeyTypeCVRef, typename KeyType = uncvref_t, + bool = is_json_pointer::value || is_json_iterator_of::value > +struct is_string_view_convertible_key_type : std::false_type {}; + +template +struct is_string_view_convertible_key_type + : std::integral_constant < bool, + std::is_convertible::value + && !is_usable_as_key_type::value + && is_usable_as_key_type::value > {}; +#else +template +struct is_string_view_convertible_key_type : std::false_type {}; +#endif + // type trait to check if KeyType can be used as an object key // true if: // - KeyType is comparable with BasicJsonType::object_t::key_type @@ -773,9 +797,7 @@ using is_usable_as_basic_json_key_type = typename std::conditional < typename BasicJsonType::object_t::key_type, KeyTypeCVRef, RequireTransparentComparator, ExcludeObjectKeyType>::value && !is_json_iterator_of::value) -#ifdef JSON_HAS_CPP_17 - || std::is_convertible::value -#endif + || is_string_view_convertible_key_type::value , std::true_type, std::false_type >::type; diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index bf3052748..f83c29480 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -813,6 +813,24 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec return it; } + /// @brief the key to look up an object member with: the key itself, or its + /// std::string_view if the object can only be searched with that + template < typename KeyType, detail::enable_if_t < + !detail::is_string_view_convertible_key_type::value, int > = 0 > + static KeyType && lookup_key(KeyType && key) noexcept + { + return std::forward(key); + } + +#ifdef JSON_HAS_CPP_17 + template < typename KeyType, detail::enable_if_t < + detail::is_string_view_convertible_key_type::value, int > = 0 > + static std::string_view lookup_key(KeyType && key) + { + return std::forward(key); + } +#endif + /// @brief erase an element from the object and return the following one /// Not every map returns an iterator from erase(iterator): some containers /// (e.g., Abseil's hash maps) return void to avoid computing a successor @@ -3008,7 +3026,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec JSON_THROW(type_error::create(304, detail::concat("cannot use at() with ", type_name()), this)); } - auto it = m_data.m_value.object->find(std::forward(key)); + auto it = m_data.m_value.object->find(lookup_key(std::forward(key))); if (it == m_data.m_value.object->end()) { JSON_THROW(out_of_range::create(403, detail::concat("key '", string_t(std::forward(key)), "' not found"), this)); @@ -3046,7 +3064,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec JSON_THROW(type_error::create(304, detail::concat("cannot use at() with ", type_name()), this)); } - auto it = m_data.m_value.object->find(std::forward(key)); + auto it = m_data.m_value.object->find(lookup_key(std::forward(key))); if (it == m_data.m_value.object->end()) { JSON_THROW(out_of_range::create(403, detail::concat("key '", string_t(std::forward(key)), "' not found"), this)); @@ -3190,7 +3208,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // operator[] only works for objects if (JSON_HEDLEY_LIKELY(is_object())) { - auto result = m_data.m_value.object->emplace(std::forward(key), nullptr); + auto result = m_data.m_value.object->emplace(lookup_key(std::forward(key)), nullptr); return set_parent(result.first->second); } @@ -3206,7 +3224,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // const operator[] only works for objects if (JSON_HEDLEY_LIKELY(is_object())) { - auto it = m_data.m_value.object->find(std::forward(key)); + auto it = m_data.m_value.object->find(lookup_key(std::forward(key))); JSON_ASSERT(it != m_data.m_value.object->end()); return it->second; } @@ -3216,8 +3234,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec private: template - using is_comparable_with_object_key = detail::is_comparable < - object_comparator_t, const typename object_t::key_type&, KeyType >; + using is_comparable_with_object_key = std::integral_constant < bool, + detail::is_comparable < + object_comparator_t, const typename object_t::key_type&, KeyType >::value + || detail::is_string_view_convertible_key_type::value >; template using value_return_type = std::conditional < @@ -3604,7 +3624,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec JSON_THROW(type_error::create(307, detail::concat("cannot use erase() with ", type_name()), this)); } - const auto it = m_data.m_value.object->find(std::forward(key)); + const auto it = m_data.m_value.object->find(lookup_key(std::forward(key))); if (it != m_data.m_value.object->end()) { m_data.m_value.object->erase(it); @@ -3631,7 +3651,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec detail::is_usable_as_basic_json_key_type::value, int> = 0> size_type erase(KeyType && key) { - return erase_internal(std::forward(key)); + return erase_internal(lookup_key(std::forward(key))); } /// @brief remove element from a JSON array given an index @@ -3711,7 +3731,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec if (is_object()) { - result.m_it.object_iterator = m_data.m_value.object->find(std::forward(key)); + result.m_it.object_iterator = m_data.m_value.object->find(lookup_key(std::forward(key))); } return result; @@ -3727,7 +3747,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec if (is_object()) { - result.m_it.object_iterator = m_data.m_value.object->find(std::forward(key)); + result.m_it.object_iterator = m_data.m_value.object->find(lookup_key(std::forward(key))); } return result; @@ -3750,7 +3770,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec size_type count(KeyType && key) const { // return 0 for all nonobject types - return is_object() ? m_data.m_value.object->count(std::forward(key)) : 0; + return is_object() ? m_data.m_value.object->count(lookup_key(std::forward(key))) : 0; } /// @brief check the existence of an element in a JSON object @@ -3768,7 +3788,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec JSON_HEDLEY_WARN_UNUSED_RESULT bool contains(KeyType && key) const { - return is_object() && m_data.m_value.object->find(std::forward(key)) != m_data.m_value.object->end(); + return is_object() && m_data.m_value.object->find(lookup_key(std::forward(key))) != m_data.m_value.object->end(); } /// @brief check the existence of an element in a JSON object given a JSON pointer diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 3c1048003..689e233bd 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -4760,6 +4760,30 @@ using is_usable_as_key_type = typename std::conditional < std::true_type, std::false_type >::type; +#ifdef JSON_HAS_CPP_17 +// type trait to check if KeyType can only be used as an object key after +// converting it to std::string_view: it is convertible to std::string_view, the +// object's comparator cannot compare it with object_t::key_type directly, but +// can compare a std::string_view. JSON pointers and JSON iterators are ruled out +// first, so that the conversion checks are never instantiated for them (a JSON +// pointer's deprecated conversion to string_t would be named otherwise). +template < typename BasicJsonType, typename KeyTypeCVRef, typename KeyType = uncvref_t, + bool = is_json_pointer::value || is_json_iterator_of::value > +struct is_string_view_convertible_key_type : std::false_type {}; + +template +struct is_string_view_convertible_key_type + : std::integral_constant < bool, + std::is_convertible::value + && !is_usable_as_key_type::value + && is_usable_as_key_type::value > {}; +#else +template +struct is_string_view_convertible_key_type : std::false_type {}; +#endif + // type trait to check if KeyType can be used as an object key // true if: // - KeyType is comparable with BasicJsonType::object_t::key_type @@ -4773,9 +4797,7 @@ using is_usable_as_basic_json_key_type = typename std::conditional < typename BasicJsonType::object_t::key_type, KeyTypeCVRef, RequireTransparentComparator, ExcludeObjectKeyType>::value && !is_json_iterator_of::value) -#ifdef JSON_HAS_CPP_17 - || std::is_convertible::value -#endif + || is_string_view_convertible_key_type::value , std::true_type, std::false_type >::type; @@ -27521,6 +27543,24 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec return it; } + /// @brief the key to look up an object member with: the key itself, or its + /// std::string_view if the object can only be searched with that + template < typename KeyType, detail::enable_if_t < + !detail::is_string_view_convertible_key_type::value, int > = 0 > + static KeyType && lookup_key(KeyType && key) noexcept + { + return std::forward(key); + } + +#ifdef JSON_HAS_CPP_17 + template < typename KeyType, detail::enable_if_t < + detail::is_string_view_convertible_key_type::value, int > = 0 > + static std::string_view lookup_key(KeyType && key) + { + return std::forward(key); + } +#endif + /// @brief erase an element from the object and return the following one /// Not every map returns an iterator from erase(iterator): some containers /// (e.g., Abseil's hash maps) return void to avoid computing a successor @@ -29716,7 +29756,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec JSON_THROW(type_error::create(304, detail::concat("cannot use at() with ", type_name()), this)); } - auto it = m_data.m_value.object->find(std::forward(key)); + auto it = m_data.m_value.object->find(lookup_key(std::forward(key))); if (it == m_data.m_value.object->end()) { JSON_THROW(out_of_range::create(403, detail::concat("key '", string_t(std::forward(key)), "' not found"), this)); @@ -29754,7 +29794,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec JSON_THROW(type_error::create(304, detail::concat("cannot use at() with ", type_name()), this)); } - auto it = m_data.m_value.object->find(std::forward(key)); + auto it = m_data.m_value.object->find(lookup_key(std::forward(key))); if (it == m_data.m_value.object->end()) { JSON_THROW(out_of_range::create(403, detail::concat("key '", string_t(std::forward(key)), "' not found"), this)); @@ -29898,7 +29938,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // operator[] only works for objects if (JSON_HEDLEY_LIKELY(is_object())) { - auto result = m_data.m_value.object->emplace(std::forward(key), nullptr); + auto result = m_data.m_value.object->emplace(lookup_key(std::forward(key)), nullptr); return set_parent(result.first->second); } @@ -29914,7 +29954,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // const operator[] only works for objects if (JSON_HEDLEY_LIKELY(is_object())) { - auto it = m_data.m_value.object->find(std::forward(key)); + auto it = m_data.m_value.object->find(lookup_key(std::forward(key))); JSON_ASSERT(it != m_data.m_value.object->end()); return it->second; } @@ -29924,8 +29964,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec private: template - using is_comparable_with_object_key = detail::is_comparable < - object_comparator_t, const typename object_t::key_type&, KeyType >; + using is_comparable_with_object_key = std::integral_constant < bool, + detail::is_comparable < + object_comparator_t, const typename object_t::key_type&, KeyType >::value + || detail::is_string_view_convertible_key_type::value >; template using value_return_type = std::conditional < @@ -30312,7 +30354,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec JSON_THROW(type_error::create(307, detail::concat("cannot use erase() with ", type_name()), this)); } - const auto it = m_data.m_value.object->find(std::forward(key)); + const auto it = m_data.m_value.object->find(lookup_key(std::forward(key))); if (it != m_data.m_value.object->end()) { m_data.m_value.object->erase(it); @@ -30339,7 +30381,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec detail::is_usable_as_basic_json_key_type::value, int> = 0> size_type erase(KeyType && key) { - return erase_internal(std::forward(key)); + return erase_internal(lookup_key(std::forward(key))); } /// @brief remove element from a JSON array given an index @@ -30419,7 +30461,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec if (is_object()) { - result.m_it.object_iterator = m_data.m_value.object->find(std::forward(key)); + result.m_it.object_iterator = m_data.m_value.object->find(lookup_key(std::forward(key))); } return result; @@ -30435,7 +30477,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec if (is_object()) { - result.m_it.object_iterator = m_data.m_value.object->find(std::forward(key)); + result.m_it.object_iterator = m_data.m_value.object->find(lookup_key(std::forward(key))); } return result; @@ -30458,7 +30500,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec size_type count(KeyType && key) const { // return 0 for all nonobject types - return is_object() ? m_data.m_value.object->count(std::forward(key)) : 0; + return is_object() ? m_data.m_value.object->count(lookup_key(std::forward(key))) : 0; } /// @brief check the existence of an element in a JSON object @@ -30476,7 +30518,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec JSON_HEDLEY_WARN_UNUSED_RESULT bool contains(KeyType && key) const { - return is_object() && m_data.m_value.object->find(std::forward(key)) != m_data.m_value.object->end(); + return is_object() && m_data.m_value.object->find(lookup_key(std::forward(key))) != m_data.m_value.object->end(); } /// @brief check the existence of an element in a JSON object given a JSON pointer diff --git a/tests/src/unit-element_access2.cpp b/tests/src/unit-element_access2.cpp index 04974c251..a64caff89 100644 --- a/tests/src/unit-element_access2.cpp +++ b/tests/src/unit-element_access2.cpp @@ -1973,4 +1973,115 @@ TEST_CASE("operator[] with user-defined std::string_view-convertible types") } } } + +TEST_CASE("keys convertible to std::string_view work with all lookup functions (regression test for #5663)") +{ + // a key type convertible only to std::string_view: the case #4958 added + // support for, but only the non-const operator[] compiled with it + struct ViewKey + { + operator std::string_view() const + { + return "a"; + } + }; + + // a key type convertible to both std::string and std::string_view: with + // 3.12.0, such a key worked with at, the const operator[], find, count and + // contains via the conversion to std::string; #4958 made the KeyType&& + // templates win overload resolution for it instead, and those then failed + struct DualKey + { + operator std::string() const + { + return "a"; + } + operator std::string_view() const + { + return "a"; + } + }; + + SECTION("nlohmann::json") + { + using json = nlohmann::json; + + SECTION("ViewKey") + { + json j = {{"a", 1}}; + const json& cj = j; + + CHECK(j[ViewKey{}] == 1); + CHECK(cj[ViewKey{}] == 1); + CHECK(j.at(ViewKey{}) == 1); + CHECK(cj.at(ViewKey{}) == 1); + CHECK(j.find(ViewKey{}) != j.end()); + CHECK(cj.find(ViewKey{}) != cj.end()); + CHECK(j.count(ViewKey{}) == 1); + CHECK(j.contains(ViewKey{})); + CHECK(j.value(ViewKey{}, 0) == 1); + CHECK(j.erase(ViewKey{}) == 1); + CHECK(!j.contains("a")); + } + + SECTION("DualKey") + { + json j = {{"a", 1}}; + const json& cj = j; + + CHECK(j[DualKey{}] == 1); + CHECK(cj[DualKey{}] == 1); + CHECK(j.at(DualKey{}) == 1); + CHECK(cj.at(DualKey{}) == 1); + CHECK(j.find(DualKey{}) != j.end()); + CHECK(cj.find(DualKey{}) != cj.end()); + CHECK(j.count(DualKey{}) == 1); + CHECK(j.contains(DualKey{})); + CHECK(j.value(DualKey{}, 0) == 1); + CHECK(j.erase(DualKey{}) == 1); + CHECK(!j.contains("a")); + } + } + + SECTION("nlohmann::ordered_json") + { + using ordered_json = nlohmann::ordered_json; + + SECTION("ViewKey") + { + ordered_json j = {{"a", 1}}; + const ordered_json& cj = j; + + CHECK(j[ViewKey{}] == 1); + CHECK(cj[ViewKey{}] == 1); + CHECK(j.at(ViewKey{}) == 1); + CHECK(cj.at(ViewKey{}) == 1); + CHECK(j.find(ViewKey{}) != j.end()); + CHECK(cj.find(ViewKey{}) != cj.end()); + CHECK(j.count(ViewKey{}) == 1); + CHECK(j.contains(ViewKey{})); + CHECK(j.value(ViewKey{}, 0) == 1); + CHECK(j.erase(ViewKey{}) == 1); + CHECK(!j.contains("a")); + } + + SECTION("DualKey") + { + ordered_json j = {{"a", 1}}; + const ordered_json& cj = j; + + CHECK(j[DualKey{}] == 1); + CHECK(cj[DualKey{}] == 1); + CHECK(j.at(DualKey{}) == 1); + CHECK(cj.at(DualKey{}) == 1); + CHECK(j.find(DualKey{}) != j.end()); + CHECK(cj.find(DualKey{}) != cj.end()); + CHECK(j.count(DualKey{}) == 1); + CHECK(j.contains(DualKey{})); + CHECK(j.value(DualKey{}, 0) == 1); + CHECK(j.erase(DualKey{}) == 1); + CHECK(!j.contains("a")); + } + } +} #endif From 49cd427196332935eb623fa448150425239d2c06 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 4 Oct 2026 11:46:36 +0200 Subject: [PATCH 08/22] Copy-construct the base class of a deep copy's elements, not assign it (#5690) The bounded-descent copy added by #5389 built the elements of a deep copy (nested past the 128-level bound) by default-constructing them and then having copy_metadata() assign their base class afterwards. That assignment is only instantiated for values nested past the bound, but being called from copy_structured() at all meant it was compiled for every copy, so a CustomBaseClass that is copy-constructible but not move-assignable (for example one with a const data member) no longer let its basic_json be copy-constructed, at any depth. copy_array_level() and copy_object_level() now build each element with a private-tag-selected constructor that copy-constructs the base class (and, under JSON_DIAGNOSTIC_POSITIONS, copies the positions) directly, the same way the copy constructor already builds elements within the 128-level bound. Copying a basic_json is therefore back to requiring only a copy-constructible base class, as documented and as it was before #5389; copy assignment is unchanged and still requires an assignable one. Fixes #5674. Signed-off-by: Niels Lohmann --- include/nlohmann/json.hpp | 65 ++++++++++++++++++-------- single_include/nlohmann/json.hpp | 65 ++++++++++++++++++-------- tests/src/unit-custom-base-class.cpp | 70 ++++++++++++++++++++++++++++ 3 files changed, 162 insertions(+), 38 deletions(-) diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index f83c29480..268c28c42 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -1028,19 +1028,39 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec using copy_scratch_value_t = std::pair; using copy_scratch_t = std::vector>; - /// @brief copy everything of @a src into @a dst but its type and value - static void copy_metadata(const basic_json& src, basic_json& dst) - { - // a custom base class is only required to be copy-constructible and - // move-assignable, so the copy has to go through a temporary - static_cast(dst) = json_base_class_t(static_cast(src)); + /// @brief tag selecting the constructor below; used only to build the + /// elements of a deep copy (@ref copy_array_level, @ref copy_object_level) + struct copy_construct_tag {}; + public: + /*! + @brief construct a null value whose base class - and, with @ref + JSON_DIAGNOSTIC_POSITIONS, positions - are copied from @a src + + Copy-constructing @ref json_base_class_t here, rather than default- + constructing the element and assigning its base class afterwards, means + that copying a @ref basic_json only ever requires a copy-constructible + base class, and never a move-assignable one as well. + + @note this constructor has to be public: @ref copy_array_level and + @ref copy_object_level reach it through @ref array_t's or @ref + object_t's own emplace_back(), which constructs the element from + outside @ref basic_json and so cannot call a private constructor. + @ref copy_construct_tag is private, though, and nothing in the + public interface hands out a value of it, so outside code can still + never name it to call this constructor itself. + */ + basic_json(copy_construct_tag /*unused*/, const basic_json& src) + : json_base_class_t(src) #if JSON_DIAGNOSTIC_POSITIONS - dst.start_position = src.start_position; - dst.end_position = src.end_position; + , start_position(src.start_position) + , end_position(src.end_position) #endif + { } + private: + /*! @brief copy the value of @a src into @a dst, which must not be structured @@ -1103,8 +1123,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec } /*! - @brief copy everything of @a src into the null value @a dst but the children + @brief finish the copy @a dst of @a src that a @ref copy_construct_tag + constructor started, other than the children of an object or array + @a dst already has @a src's base class and, with @ref + JSON_DIAGNOSTIC_POSITIONS, positions; only its value is still missing. Objects and arrays are not copied here; they are appended to @a worklist to be created later by @ref copy_iteratively. Until that happens, @a dst remains a null value, so that a partially built copy can be destroyed at any point @@ -1112,8 +1135,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec */ static void copy_shallow(const basic_json& src, basic_json& dst, copy_worklist_t& worklist) { - copy_metadata(src, dst); - if (src.m_data.m_type == value_t::object || src.m_data.m_type == value_t::array) { // defer: dst stays a null value until its container exists @@ -1134,15 +1155,19 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec { const array_t& src_array = *src.m_data.m_value.array; - // create all elements up front: growing the array afterwards could - // invalidate the pointers that are handed to the worklist; resize() - // rather than the fill constructor, because not every array type - // provides the latter (e.g., ones without a matching allocator-aware - // fill constructor) dst.m_data.m_value.array = create(); // only now that the array exists may dst stop being a null value dst.m_data.m_type = value_t::array; - dst.m_data.m_value.array->resize(src_array.size()); + + // create every element - its base class already copy-constructed from + // its counterpart in src, via the copy_construct_tag constructor - + // before any of their addresses are handed to worklist below: growing + // the array while that is going on could reallocate it and invalidate + // addresses taken from an earlier iteration + for (const auto& src_element : src_array) + { + dst.m_data.m_value.array->emplace_back(copy_construct_tag{}, src_element); + } auto dst_it = dst.m_data.m_value.array->begin(); for (auto src_it = src_array.cbegin(); src_it != src_array.cend(); ++src_it, ++dst_it) @@ -1160,12 +1185,14 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // build the complete key skeleton and hand it to the object's range // constructor: adding the keys one by one would be quadratic for object - // types that are backed by a vector, such as nlohmann::ordered_map + // types that are backed by a vector, such as nlohmann::ordered_map; each + // value's base class is already copy-constructed from its counterpart + // in src, via the copy_construct_tag constructor scratch.clear(); scratch.reserve(src_object.size()); for (const auto& element : src_object) { - scratch.emplace_back(element.first, basic_json()); + scratch.emplace_back(element.first, basic_json(copy_construct_tag{}, element.second)); } dst.m_data.m_value.object = create(std::make_move_iterator(scratch.begin()), diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 689e233bd..349977636 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -27758,19 +27758,39 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec using copy_scratch_value_t = std::pair; using copy_scratch_t = std::vector>; - /// @brief copy everything of @a src into @a dst but its type and value - static void copy_metadata(const basic_json& src, basic_json& dst) - { - // a custom base class is only required to be copy-constructible and - // move-assignable, so the copy has to go through a temporary - static_cast(dst) = json_base_class_t(static_cast(src)); + /// @brief tag selecting the constructor below; used only to build the + /// elements of a deep copy (@ref copy_array_level, @ref copy_object_level) + struct copy_construct_tag {}; + public: + /*! + @brief construct a null value whose base class - and, with @ref + JSON_DIAGNOSTIC_POSITIONS, positions - are copied from @a src + + Copy-constructing @ref json_base_class_t here, rather than default- + constructing the element and assigning its base class afterwards, means + that copying a @ref basic_json only ever requires a copy-constructible + base class, and never a move-assignable one as well. + + @note this constructor has to be public: @ref copy_array_level and + @ref copy_object_level reach it through @ref array_t's or @ref + object_t's own emplace_back(), which constructs the element from + outside @ref basic_json and so cannot call a private constructor. + @ref copy_construct_tag is private, though, and nothing in the + public interface hands out a value of it, so outside code can still + never name it to call this constructor itself. + */ + basic_json(copy_construct_tag /*unused*/, const basic_json& src) + : json_base_class_t(src) #if JSON_DIAGNOSTIC_POSITIONS - dst.start_position = src.start_position; - dst.end_position = src.end_position; + , start_position(src.start_position) + , end_position(src.end_position) #endif + { } + private: + /*! @brief copy the value of @a src into @a dst, which must not be structured @@ -27833,8 +27853,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec } /*! - @brief copy everything of @a src into the null value @a dst but the children + @brief finish the copy @a dst of @a src that a @ref copy_construct_tag + constructor started, other than the children of an object or array + @a dst already has @a src's base class and, with @ref + JSON_DIAGNOSTIC_POSITIONS, positions; only its value is still missing. Objects and arrays are not copied here; they are appended to @a worklist to be created later by @ref copy_iteratively. Until that happens, @a dst remains a null value, so that a partially built copy can be destroyed at any point @@ -27842,8 +27865,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec */ static void copy_shallow(const basic_json& src, basic_json& dst, copy_worklist_t& worklist) { - copy_metadata(src, dst); - if (src.m_data.m_type == value_t::object || src.m_data.m_type == value_t::array) { // defer: dst stays a null value until its container exists @@ -27864,15 +27885,19 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec { const array_t& src_array = *src.m_data.m_value.array; - // create all elements up front: growing the array afterwards could - // invalidate the pointers that are handed to the worklist; resize() - // rather than the fill constructor, because not every array type - // provides the latter (e.g., ones without a matching allocator-aware - // fill constructor) dst.m_data.m_value.array = create(); // only now that the array exists may dst stop being a null value dst.m_data.m_type = value_t::array; - dst.m_data.m_value.array->resize(src_array.size()); + + // create every element - its base class already copy-constructed from + // its counterpart in src, via the copy_construct_tag constructor - + // before any of their addresses are handed to worklist below: growing + // the array while that is going on could reallocate it and invalidate + // addresses taken from an earlier iteration + for (const auto& src_element : src_array) + { + dst.m_data.m_value.array->emplace_back(copy_construct_tag{}, src_element); + } auto dst_it = dst.m_data.m_value.array->begin(); for (auto src_it = src_array.cbegin(); src_it != src_array.cend(); ++src_it, ++dst_it) @@ -27890,12 +27915,14 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // build the complete key skeleton and hand it to the object's range // constructor: adding the keys one by one would be quadratic for object - // types that are backed by a vector, such as nlohmann::ordered_map + // types that are backed by a vector, such as nlohmann::ordered_map; each + // value's base class is already copy-constructed from its counterpart + // in src, via the copy_construct_tag constructor scratch.clear(); scratch.reserve(src_object.size()); for (const auto& element : src_object) { - scratch.emplace_back(element.first, basic_json()); + scratch.emplace_back(element.first, basic_json(copy_construct_tag{}, element.second)); } dst.m_data.m_value.object = create(std::make_move_iterator(scratch.begin()), diff --git a/tests/src/unit-custom-base-class.cpp b/tests/src/unit-custom-base-class.cpp index 138940d3d..a6b9b9ea4 100644 --- a/tests/src/unit-custom-base-class.cpp +++ b/tests/src/unit-custom-base-class.cpp @@ -405,3 +405,73 @@ TEST_CASE("JSON Visit Node") ); CHECK(expected.empty()); } + +// A custom base class with a const member: copy-constructible (initializing a +// const member works fine), but not copy-/move-assignable (assigning one does +// not). Used to check that copy construction never requires more than that. +struct const_member_base +{ + const int id = 7; // NOLINT(misc-non-private-member-variables-in-classes) +}; + +using json_with_const_base = nlohmann::basic_json < + std::map, + std::vector, + std::string, + bool, + std::int64_t, + std::uint64_t, + double, + std::allocator, + nlohmann::adl_serializer, + std::vector, + const_member_base + >; + +// build an array nested @a depth levels deep, with the innermost value 1; +// every level is constructed (never assigned), since const_member_base does +// not support assignment +static json_with_const_base make_nested_array(std::size_t depth) +{ + if (depth == 0) + { + return json_with_const_base(1); + } + return json_with_const_base::array({make_nested_array(depth - 1)}); +} + +TEST_CASE("Regression test for issue #5674 - copy construction must not require an assignable base class") +{ + SECTION("depth 0") + { + // as in the original bug report: copy construction only, no assignment + const json_with_const_base j = {1, 2}; + const json_with_const_base copy = j; // NOLINT(performance-unnecessary-copy-initialization) + + CHECK(copy.size() == 2); + CHECK(copy.id == 7); + } + + SECTION("nested deeper than the copy constructor's descent bound") + { + // beyond nesting_depth_limit() (128) levels, the copy constructor + // copies without the call stack (copy_iteratively / copy_array_level), + // which used to assign the base class of every element it created + const std::size_t depth = 300; + + const json_with_const_base j = make_nested_array(depth); + const json_with_const_base copy = j; // NOLINT(performance-unnecessary-copy-initialization) + + const json_with_const_base* c = © + for (std::size_t level = 0; level <= depth; ++level) + { + CAPTURE(level) + REQUIRE(c->id == 7); + if (level < depth) + { + c = &c->at(0); + } + } + CHECK(*c == 1); + } +} From 756b28c2b85d668c9e74e3fc3c4688039ebc2901 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 4 Oct 2026 11:46:39 +0200 Subject: [PATCH 09/22] Keep a NUL byte ending a // comment as the end of input (#5696) With the default NUL handling (JSON_STRICT_NUL_HANDLING not set), a NUL byte in the input is treated as the real end of input everywhere - except when it immediately ends a `//` comment: scan_comment() matched '\0' as a comment terminator like '\n', so the NUL was consumed as part of the comment and scan() never saw it as end of input; the next get() then kept reading past it. Multi-line comments and JSON_STRICT_NUL_HANDLING=1 were unaffected, since there the NUL is just part of the comment text. Fix scan_comment() to leave the NUL unconsumed (unget()) instead of returning it as part of the comment, so the following scan() reports it as end of input, exactly as for a NUL anywhere else. Fixes #5659. Signed-off-by: Niels Lohmann --- include/nlohmann/detail/input/lexer.hpp | 7 ++++- single_include/nlohmann/json.hpp | 7 ++++- tests/src/unit-class_parser.cpp | 39 +++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 2 deletions(-) diff --git a/include/nlohmann/detail/input/lexer.hpp b/include/nlohmann/detail/input/lexer.hpp index a67a0228c..052bc4de9 100644 --- a/include/nlohmann/detail/input/lexer.hpp +++ b/include/nlohmann/detail/input/lexer.hpp @@ -941,10 +941,15 @@ class lexer : public lexer_base case '\n': case '\r': case char_traits::eof(): + return true; + #if !JSON_STRICT_NUL_HANDLING case '\0': -#endif + // a NUL byte is the end of the input (see scan()), + // so leave it for scan() to see + unget(); return true; +#endif default: break; diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 349977636..884430274 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -11027,10 +11027,15 @@ class lexer : public lexer_base case '\n': case '\r': case char_traits::eof(): + return true; + #if !JSON_STRICT_NUL_HANDLING case '\0': -#endif + // a NUL byte is the end of the input (see scan()), + // so leave it for scan() to see + unget(); return true; +#endif default: break; diff --git a/tests/src/unit-class_parser.cpp b/tests/src/unit-class_parser.cpp index f67ba631b..75f3757e8 100644 --- a/tests/src/unit-class_parser.cpp +++ b/tests/src/unit-class_parser.cpp @@ -592,6 +592,45 @@ TEST_CASE("parser class") // parsing from a string literal is unaffected either way CHECK(json::parse("123") == json(123)); + + // a NUL byte that ends a // comment ends the input just + // like a NUL byte anywhere else (issue #5659); before the + // fix, the NUL was consumed as part of the comment, and + // scanning continued with whatever followed it + { + // same as "//c" alone (real end of input after the + // comment), rather than continuing with "[1]" + std::string s1 = "//c"; + s1.push_back('\0'); + s1 += "[1]"; + json _; // NOLINT(readability-identifier-naming) + CHECK_THROWS_WITH_AS(_ = json::parse(s1, nullptr, true, true), + "[json.exception.parse_error.101] parse error at line 1, column 4: syntax error while parsing value - unexpected end of input; expected '[', '{', or a literal", + json::parse_error&); + CHECK_FALSE(json::accept(s1, true, true)); + } + + { + // same as "[1, //c" alone, rather than continuing with " 2]" + std::string s2 = "[1, //c"; + s2.push_back('\0'); + s2 += " 2]"; + json _; // NOLINT(readability-identifier-naming) + CHECK_THROWS_WITH_AS(_ = json::parse(s2, nullptr, true, true), + "[json.exception.parse_error.101] parse error at line 1, column 8: syntax error while parsing value - unexpected end of input; expected '[', '{', or a literal", + json::parse_error&); + CHECK_FALSE(json::accept(s2, true, true)); + } + + { + // same as "1 //c" alone: the comment (and the NUL that + // ends it) is ignored, and "x" is never reached + std::string s3 = "1 //c"; + s3.push_back('\0'); + s3 += "x"; + CHECK(json::parse(s3, nullptr, true, true) == json(1)); + CHECK(json::accept(s3, true, true)); + } } #endif From 1edf0ef041fe20710cea8d236a4bab05d53b0fce Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 4 Oct 2026 11:46:43 +0200 Subject: [PATCH 10/22] Fix value(json_pointer, default) aborting under JSON_NOEXCEPTION (#5700) With exceptions disabled (JSON_NOEXCEPTION or -fno-exceptions), value(const json_pointer&, default) called std::abort() for array reference tokens that array_index() rejects with out_of_range.404/410: indices too large to fit size_type, the empty token ("/"), and tokens like "/1a". With exceptions enabled, the same tokens correctly yielded the default value, because get_checked_or_null() relied on JSON_TRY/JSON_INTERNAL_CATCH (detail::out_of_range&) to turn the exception into nullptr; under JSON_NOEXCEPTION, JSON_THROW aborts before that catch is ever reached. get_checked_or_null() now detects those out-of-range tokens itself, the same way contains(json_pointer) already does (#5495), and only calls array_index() for tokens that must still raise parse_error.106 or parse_error.109 (e.g. "/01", "/+1"), matching the documented behavior of value(). Added regression tests to tests/src/unit-disabled_exceptions.cpp (built with JSON_NOEXCEPTION and -fno-exceptions) and the matching checks to tests/src/unit-element_access2.cpp for normal exception mode. Fixes #5672. Signed-off-by: Niels Lohmann --- include/nlohmann/detail/json_pointer.hpp | 26 ++++++++++++++++-------- single_include/nlohmann/json.hpp | 26 ++++++++++++++++-------- tests/src/unit-disabled_exceptions.cpp | 15 ++++++++++++++ tests/src/unit-element_access2.cpp | 15 ++++++++++++++ 4 files changed, 66 insertions(+), 16 deletions(-) diff --git a/include/nlohmann/detail/json_pointer.hpp b/include/nlohmann/detail/json_pointer.hpp index e34acb6d3..a10f4f39f 100644 --- a/include/nlohmann/detail/json_pointer.hpp +++ b/include/nlohmann/detail/json_pointer.hpp @@ -685,19 +685,29 @@ class json_pointer return nullptr; } - // may throw parse_error.106/109 for a malformed index; an - // index that is syntactically valid but cannot be - // represented (out_of_range.404/410) is treated like an - // out-of-range index below - typename BasicJsonType::size_type idx{}; - JSON_TRY + // tokens that array_index() rejects with parse_error.106/109 + // are passed on to it; all other tokens that it would reject + // with out_of_range.404/410 are detected here, so that this + // also works without exceptions + if (JSON_HEDLEY_UNLIKELY(reference_token.size() > 1 && !(reference_token[0] >= '1' && reference_token[0] <= '9'))) { - idx = array_index(reference_token); + static_cast(array_index(reference_token)); // throws parse_error.106/109 } - JSON_INTERNAL_CATCH (detail::out_of_range&) + if (JSON_HEDLEY_UNLIKELY(reference_token.empty() || !std::all_of(reference_token.begin(), reference_token.end(), [](const char c) + { + return c >= '0' && c <= '9'; + }))) { return nullptr; } + errno = 0; // strtoull() does not reset errno on success + char* p_end = nullptr; // NOLINT(misc-const-correctness) + const unsigned long long magnitude = std::strtoull(reference_token.data(), &p_end, 10); // NOLINT(runtime/int) + if (JSON_HEDLEY_UNLIKELY(errno == ERANGE || magnitude >= static_cast((std::numeric_limits::max)()))) // NOLINT(runtime/int) + { + return nullptr; + } + const auto idx = static_cast(magnitude); if (JSON_HEDLEY_UNLIKELY(idx >= ptr->m_data.m_value.array->size())) { diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 884430274..1ec6e9cc5 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -20281,19 +20281,29 @@ class json_pointer return nullptr; } - // may throw parse_error.106/109 for a malformed index; an - // index that is syntactically valid but cannot be - // represented (out_of_range.404/410) is treated like an - // out-of-range index below - typename BasicJsonType::size_type idx{}; - JSON_TRY + // tokens that array_index() rejects with parse_error.106/109 + // are passed on to it; all other tokens that it would reject + // with out_of_range.404/410 are detected here, so that this + // also works without exceptions + if (JSON_HEDLEY_UNLIKELY(reference_token.size() > 1 && !(reference_token[0] >= '1' && reference_token[0] <= '9'))) { - idx = array_index(reference_token); + static_cast(array_index(reference_token)); // throws parse_error.106/109 } - JSON_INTERNAL_CATCH (detail::out_of_range&) + if (JSON_HEDLEY_UNLIKELY(reference_token.empty() || !std::all_of(reference_token.begin(), reference_token.end(), [](const char c) + { + return c >= '0' && c <= '9'; + }))) { return nullptr; } + errno = 0; // strtoull() does not reset errno on success + char* p_end = nullptr; // NOLINT(misc-const-correctness) + const unsigned long long magnitude = std::strtoull(reference_token.data(), &p_end, 10); // NOLINT(runtime/int) + if (JSON_HEDLEY_UNLIKELY(errno == ERANGE || magnitude >= static_cast((std::numeric_limits::max)()))) // NOLINT(runtime/int) + { + return nullptr; + } + const auto idx = static_cast(magnitude); if (JSON_HEDLEY_UNLIKELY(idx >= ptr->m_data.m_value.array->size())) { diff --git a/tests/src/unit-disabled_exceptions.cpp b/tests/src/unit-disabled_exceptions.cpp index 0b8de64d3..8e3adf944 100644 --- a/tests/src/unit-disabled_exceptions.cpp +++ b/tests/src/unit-disabled_exceptions.cpp @@ -47,6 +47,21 @@ TEST_CASE("Tests with disabled exceptions") delete sax_no_exception::error_string; // NOLINT(cppcoreguidelines-owning-memory) } + SECTION("issue #5672 - value(json_pointer, default) must not abort for array tokens that are not a valid index") + { + const json j = {1, 2, 3}; + + // a syntactically valid index that is out of range for this array + CHECK(j.value("/7"_json_pointer, 42) == 42); + // a reference token that is not a number at all + CHECK(j.value("/1a"_json_pointer, 42) == 42); + // the empty reference token (JSON pointer "/") + CHECK(j.value("/"_json_pointer, 42) == 42); + // an index whose magnitude does not fit into size_type + CHECK(j.value("/99999999999999999999999"_json_pointer, 42) == 42); + CHECK(j.value("/18446744073709551615"_json_pointer, 42) == 42); + } + SECTION("growing an ordered_json object") { auto j = nlohmann::ordered_json::object(); diff --git a/tests/src/unit-element_access2.cpp b/tests/src/unit-element_access2.cpp index a64caff89..efd7b15a0 100644 --- a/tests/src/unit-element_access2.cpp +++ b/tests/src/unit-element_access2.cpp @@ -516,6 +516,21 @@ TEST_CASE_TEMPLATE("element access 2", Json, nlohmann::json, nlohmann::ordered_j CHECK(j_array.value("/-"_json_pointer, 42) == 42); CHECK(j_array_const.value("/-"_json_pointer, 42) == 42); + // Test an index with a non-digit after a valid leading digit; this is + // out_of_range (not parse_error) and must not throw (see #5672) + CHECK(j_array.value("/1a"_json_pointer, 42) == 42); + CHECK(j_array_const.value("/1a"_json_pointer, 42) == 42); + + // Test the empty reference token (JSON pointer "/"); see #5672 + CHECK(j_array.value("/"_json_pointer, 42) == 42); + CHECK(j_array_const.value("/"_json_pointer, 42) == 42); + + // Test an index whose magnitude does not fit into size_type (see #5672) + CHECK(j_array.value("/99999999999999999999999"_json_pointer, 42) == 42); + CHECK(j_array_const.value("/99999999999999999999999"_json_pointer, 42) == 42); + CHECK(j_array.value("/18446744073709551615"_json_pointer, 42) == 42); + CHECK(j_array_const.value("/18446744073709551615"_json_pointer, 42) == 42); + #if !defined(JSON_NOEXCEPTION) // Test malformed index (non-numeric) throws parse_error CHECK_THROWS_WITH_AS(j_array.value("/foo"_json_pointer, 1), "[json.exception.parse_error.109] parse error: array index 'foo' is not a number", typename Json::parse_error&); From a212d3b2e4818849a57270bb537d74564be77692 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 4 Oct 2026 11:46:47 +0200 Subject: [PATCH 11/22] Preserve the object comparator's state in a deep copy past the nesting bound (#5722) * Preserve the object comparator's state in a deep copy past the nesting bound copy_object_level(), used by the copy constructor and copy assignment once a value is nested deeper than the iterative deep copy's bound (128 levels, or every copy under JSON_NO_THREAD_LOCAL), built each object's copy with the object type's plain range constructor. That default-constructs the object's comparator instead of copying the original's. For an object type whose comparator carries state, such as a std::map that compares keys case-sensitively only when constructed that way, the copy then ordered - and could even deduplicate - its keys differently from the original. Add detail::is_comparator_constructible_object_type, a detection trait for object types that provide a key_comp() and a constructor taking a range and a comparator, the way std::map does. copy_object_level now dispatches on it: an object type that qualifies gets its copy built with src_object.key_comp() passed along; other object types, such as nlohmann::ordered_map (which has a key_compare for its std::map-like interface, but no key_comp()), keep using the plain range constructor exactly as before. merge_patch and update() were checked for the same pattern; neither is affected, since both only ever add members one at a time to an object that already has its own comparator (or start a brand new default-constructed one), rather than rebuilding an object_t from a range copied out of an existing, possibly custom-comparator object. Fixes #5649. Signed-off-by: Niels Lohmann * Keep astyle from padding the create_object_with_comparator templates Spell the negated condition as detail::negation<...> instead of a leading '!', which made astyle spread the template header out. Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann --- include/nlohmann/detail/meta/type_traits.hpp | 31 ++++++++ include/nlohmann/json.hpp | 24 +++++- single_include/nlohmann/json.hpp | 55 ++++++++++++- tests/src/unit-comparison.cpp | 81 ++++++++++++++++++++ 4 files changed, 189 insertions(+), 2 deletions(-) diff --git a/include/nlohmann/detail/meta/type_traits.hpp b/include/nlohmann/detail/meta/type_traits.hpp index 2f837e046..14029c0fc 100644 --- a/include/nlohmann/detail/meta/type_traits.hpp +++ b/include/nlohmann/detail/meta/type_traits.hpp @@ -189,6 +189,37 @@ struct actual_object_comparator template using actual_object_comparator_t = typename actual_object_comparator::type; +template +using detect_key_comp = decltype(std::declval().key_comp()); + +// whether ObjectType can be constructed from a pair of Iterator together with +// a copy of its own comparator, the way std::map can: it needs a nested +// key_compare, a const key_comp() convertible to it, and a matching +// (Iterator, Iterator, const key_compare&) constructor. +// +// used to preserve a stateful comparator when a copy is built from a range +// past the iterative deep copy's nesting bound (see copy_object_level); an +// object type that does not satisfy this, such as nlohmann::ordered_map +// (which has key_compare for its std::map-like interface, but no key_comp()), +// keeps default-constructing its comparator, just as it always has +template +struct is_comparator_constructible_object_type_impl : std::false_type {}; + +template +struct is_comparator_constructible_object_type_impl < + ObjectType, Iterator, enable_if_t::value >> +{ + using key_compare = typename ObjectType::key_compare; + + static constexpr bool value = + is_detected_convertible::value && + std::is_constructible::value; +}; + +template +struct is_comparator_constructible_object_type + : is_comparator_constructible_object_type_impl {}; + ///////////////// // char_traits // ///////////////// diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 268c28c42..c8c9854ca 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -1176,6 +1176,27 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec } } + /// @brief create the object type from a range, preserving @a src_object's + /// comparator when the object type supports it + /// Enabled for object types that provide a key_comp() and a matching + /// range-plus-comparator constructor, such as std::map. Other object + /// types, such as nlohmann::ordered_map, fall back to the plain range + /// constructor and default-construct their comparator, just as they + /// always have (@ref detail::is_comparator_constructible_object_type). + template::value, int> = 0> + static object_t* create_object_with_comparator(const object_t& src_object, Iterator first, Iterator last) + { + return create(first, last, src_object.key_comp()); + } + + template>::value, int> = 0> + static object_t* create_object_with_comparator(const object_t& /*src_object*/, Iterator first, Iterator last) + { + return create(first, last); + } + /// @brief create the copy of the object @a src in @a dst /// @note structured values are appended to @a worklist instead static void copy_object_level(const basic_json& src, basic_json& dst, @@ -1195,7 +1216,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec scratch.emplace_back(element.first, basic_json(copy_construct_tag{}, element.second)); } - dst.m_data.m_value.object = create(std::make_move_iterator(scratch.begin()), + dst.m_data.m_value.object = create_object_with_comparator(src_object, + std::make_move_iterator(scratch.begin()), std::make_move_iterator(scratch.end())); // only now that the object exists may dst stop being a null value dst.m_data.m_type = value_t::object; diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 1ec6e9cc5..84ad40c74 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -4189,6 +4189,37 @@ struct actual_object_comparator template using actual_object_comparator_t = typename actual_object_comparator::type; +template +using detect_key_comp = decltype(std::declval().key_comp()); + +// whether ObjectType can be constructed from a pair of Iterator together with +// a copy of its own comparator, the way std::map can: it needs a nested +// key_compare, a const key_comp() convertible to it, and a matching +// (Iterator, Iterator, const key_compare&) constructor. +// +// used to preserve a stateful comparator when a copy is built from a range +// past the iterative deep copy's nesting bound (see copy_object_level); an +// object type that does not satisfy this, such as nlohmann::ordered_map +// (which has key_compare for its std::map-like interface, but no key_comp()), +// keeps default-constructing its comparator, just as it always has +template +struct is_comparator_constructible_object_type_impl : std::false_type {}; + +template +struct is_comparator_constructible_object_type_impl < + ObjectType, Iterator, enable_if_t::value >> +{ + using key_compare = typename ObjectType::key_compare; + + static constexpr bool value = + is_detected_convertible::value && + std::is_constructible::value; +}; + +template +struct is_comparator_constructible_object_type + : is_comparator_constructible_object_type_impl {}; + ///////////////// // char_traits // ///////////////// @@ -27921,6 +27952,27 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec } } + /// @brief create the object type from a range, preserving @a src_object's + /// comparator when the object type supports it + /// Enabled for object types that provide a key_comp() and a matching + /// range-plus-comparator constructor, such as std::map. Other object + /// types, such as nlohmann::ordered_map, fall back to the plain range + /// constructor and default-construct their comparator, just as they + /// always have (@ref detail::is_comparator_constructible_object_type). + template::value, int> = 0> + static object_t* create_object_with_comparator(const object_t& src_object, Iterator first, Iterator last) + { + return create(first, last, src_object.key_comp()); + } + + template>::value, int> = 0> + static object_t* create_object_with_comparator(const object_t& /*src_object*/, Iterator first, Iterator last) + { + return create(first, last); + } + /// @brief create the copy of the object @a src in @a dst /// @note structured values are appended to @a worklist instead static void copy_object_level(const basic_json& src, basic_json& dst, @@ -27940,7 +27992,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec scratch.emplace_back(element.first, basic_json(copy_construct_tag{}, element.second)); } - dst.m_data.m_value.object = create(std::make_move_iterator(scratch.begin()), + dst.m_data.m_value.object = create_object_with_comparator(src_object, + std::make_move_iterator(scratch.begin()), std::make_move_iterator(scratch.end())); // only now that the object exists may dst stop being a null value dst.m_data.m_type = value_t::object; diff --git a/tests/src/unit-comparison.cpp b/tests/src/unit-comparison.cpp index 4b66f1d07..15715fec2 100644 --- a/tests/src/unit-comparison.cpp +++ b/tests/src/unit-comparison.cpp @@ -849,6 +849,46 @@ Json nest(Json j, const std::size_t depth) return j; } +// a std::map comparator with state: case-insensitive, unless constructed +// case-sensitive. Used to check that copying an object copies the original's +// comparator rather than default-constructing a new one (see #5649). +struct key_case_less +{ + key_case_less() = default; + explicit key_case_less(const bool cs) noexcept : case_sensitive(cs) {} + + bool operator()(const std::string& a, const std::string& b) const + { + if (case_sensitive) + { + return a < b; + } + return std::lexicographical_compare(a.begin(), a.end(), b.begin(), b.end(), + [](unsigned char x, unsigned char y) + { + return std::tolower(x) < std::tolower(y); + }); + } + + bool case_sensitive = false; +}; + +template +using key_case_map = std::map; +using key_case_json = nlohmann::basic_json; + +// the innermost value of a chain of single-element arrays +template +const Json& innermost(const Json& j) +{ + const Json* p = &j; + while (p->is_array()) + { + p = &(*p)[0]; + } + return *p; +} + // orders keys case-insensitively, so "key" and "KEY" compare equivalent // (neither less than the other) although they are not equal struct case_insensitive_less @@ -913,6 +953,47 @@ TEST_CASE("equality of objects whose entries have no fixed order") } } +TEST_CASE("copying an object preserves its comparator's state") +{ + // Past the iterative deep copy's nesting bound, an object copy used to be + // built with a default-constructed comparator instead of a copy of the + // original's. For an object type whose comparator carries state - here, a + // std::map that compares keys case-sensitively only when created that way + // - this reordered the copy's keys and could even drop entries that the + // original's comparator kept distinct (see #5649). + key_case_json object = key_case_json::object_t(key_case_less(true)); // case-sensitive + object["b"] = 1; + object["B"] = 2; + object["a"] = 3; + REQUIRE(object.dump() == R"({"B":2,"a":3,"b":1})"); + + for (const std::size_t depth : std::vector {0, 127, 128, 200}) + { + CAPTURE(depth); + + key_case_json original = object; + for (std::size_t i = 0; i < depth; ++i) + { + original = key_case_json::array({std::move(original)}); + } + + { + const key_case_json copy = original; // NOLINT(performance-unnecessary-copy-initialization) + CHECK(innermost(copy).size() == 3); + CHECK(innermost(copy).dump() == R"({"B":2,"a":3,"b":1})"); + CHECK(copy == original); + } + + { + key_case_json copy = key_case_json::array(); + copy = original; + CHECK(innermost(copy).size() == 3); + CHECK(innermost(copy).dump() == R"({"B":2,"a":3,"b":1})"); + CHECK(copy == original); + } + } +} + TEST_CASE("equality of an object whose comparator treats different keys as equivalent") { // https://github.com/nlohmann/json/issues/5655: past the nesting bound, From 1df1e8a845350f1156f8b4ac24887d3f40b4e71a Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 4 Oct 2026 11:46:52 +0200 Subject: [PATCH 12/22] Make cross-string-type basic_json conversion explicit without implicit conversions (#5591) * Make cross-string-type basic_json conversion explicit without implicit conversions The converting constructor from another basic_json specialization was always implicit, so a value with a different string_t (std::wstring, a string with a custom allocator, ...) silently converted into a temporary, e.g. when passed to a function taking const nlohmann::json&. Such conversions do not produce correct values (#3425), and JSON_USE_IMPLICIT_CONVERSIONS=0 did not catch them. When JSON_USE_IMPLICIT_CONVERSIONS is 0, the constructor is now explicit if the string types differ. Specializations sharing a string type (json and ordered_json, different serializers or object maps) stay implicitly convertible, so the NLOHMANN_DEFINE_TYPE_* macros keep working with nested json members. get() constructs explicitly and works in both modes. Fixes #2649. Signed-off-by: Niels Lohmann * Construct explicitly in get_to() and to_json(std::optional) With JSON_USE_IMPLICIT_CONVERSIONS=0 the conversion from a basic_json with a different string type is now explicit, but two library paths still assigned such a value implicitly and failed to compile inside the library: - get_to() with a basic_json target (the #2175 overload) did `v = *this`, so json(42).get_to(alt_json&) broke although get() works. - to_json(BasicJsonType&, const std::optional&) is constrained on std::is_constructible (which accepts the explicit constructor) but did `j = *opt`, so converting a std::optional into a json broke. Both now construct the value explicitly, as get_impl() already does. Also replace static_cast(JSON_USE_IMPLICIT_CONVERSIONS) with a comparison: clang-tidy's modernize-use-bool-literals rejected the cast of the integer literal the macro expands to, failing ci_clang_tidy. Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann --- docs/mkdocs/docs/api/basic_json/basic_json.md | 12 +++++- .../macros/json_use_implicit_conversions.md | 24 ++++++++++- .../nlohmann/detail/conversions/to_json.hpp | 4 +- include/nlohmann/json.hpp | 37 +++++++++++++++-- single_include/nlohmann/json.hpp | 41 +++++++++++++++++-- tests/src/unit-alt-string.cpp | 36 ++++++++++++++++ 6 files changed, 144 insertions(+), 10 deletions(-) diff --git a/docs/mkdocs/docs/api/basic_json/basic_json.md b/docs/mkdocs/docs/api/basic_json/basic_json.md index 1a0b101bd..78fa3560c 100644 --- a/docs/mkdocs/docs/api/basic_json/basic_json.md +++ b/docs/mkdocs/docs/api/basic_json/basic_json.md @@ -293,6 +293,15 @@ basic_json(basic_json&& other) noexcept; When used without parentheses around an empty initializer list, `basic_json()` is called instead of this function, yielding the JSON `#!json null` value. +- Overload 4: + + !!! info "Implicit conversion" + + The conversion is implicit unless [`JSON_USE_IMPLICIT_CONVERSIONS`](../macros/json_use_implicit_conversions.md) + is defined to `0` and `BasicJsonType::string_t` differs from `string_t`. In that case, the constructor is + `explicit`, so a JSON value with a different string type is no longer silently converted, for example when it is + passed to a function taking `#!cpp const json&`. Write `#!cpp json(other)` or `#!cpp other.get()` instead. + - Overload 7: !!! info "Preconditions" @@ -466,7 +475,8 @@ basic_json(basic_json&& other) noexcept; 1. Since version 1.0.0. 2. Since version 1.0.0. 3. Since version 2.1.0. -4. Since version 3.2.0. +4. Since version 3.2.0. Explicit for different string types if `JSON_USE_IMPLICIT_CONVERSIONS` is `0` since + version 3.13.0. 5. Since version 1.0.0. 6. Since version 1.0.0. 7. Since version 1.0.0. Fixed in version 3.13.0 to also check the iterator range for binary values; before, a range diff --git a/docs/mkdocs/docs/api/macros/json_use_implicit_conversions.md b/docs/mkdocs/docs/api/macros/json_use_implicit_conversions.md index 5b74f1ff7..11bf74b22 100644 --- a/docs/mkdocs/docs/api/macros/json_use_implicit_conversions.md +++ b/docs/mkdocs/docs/api/macros/json_use_implicit_conversions.md @@ -5,7 +5,9 @@ ``` When defined to `0`, implicit conversions are switched off. By default, implicit conversions are switched on. The -value directly affects [`operator ValueType`](../basic_json/operator_ValueType.md). +value directly affects [`operator ValueType`](../basic_json/operator_ValueType.md) and the +[converting constructor](../basic_json/basic_json.md) from a `basic_json` specialization with a different string +type (overload 4). ## Default definition @@ -59,6 +61,25 @@ By default, implicit conversions are enabled. auto s = j.get(); ``` +??? example "Conversion between `basic_json` specializations" + + A `basic_json` specialization with a different string type is also no longer converted implicitly when + `JSON_USE_IMPLICIT_CONVERSIONS` is defined to `0`: + + ```cpp + using wjson = nlohmann::basic_json; + + void load(const nlohmann::json& j); + + wjson wj = /* ... */; + load(wj); // error: no implicit conversion + load(nlohmann::json(wj)); // OK: explicit conversion + load(wj.get()); // OK: explicit conversion + ``` + + Specializations that share the same string type, such as `json` and `ordered_json`, remain implicitly + convertible. + ## See also - [**operator ValueType**](../basic_json/operator_ValueType.md) - get a value (implicit) @@ -68,3 +89,4 @@ By default, implicit conversions are enabled. ## Version history - Added in version 3.9.0. +- Also affects the conversion between `basic_json` specializations with different string types since version 3.13.0. diff --git a/include/nlohmann/detail/conversions/to_json.hpp b/include/nlohmann/detail/conversions/to_json.hpp index 49b3c32e2..7b97068f3 100644 --- a/include/nlohmann/detail/conversions/to_json.hpp +++ b/include/nlohmann/detail/conversions/to_json.hpp @@ -294,7 +294,9 @@ void to_json(BasicJsonType& j, const std::optional& opt) noexcept(std::is_not { if (opt.has_value()) { - j = *opt; + // explicit construction, as the conversion from a basic_json with a different + // string type is explicit if JSON_USE_IMPLICIT_CONVERSIONS is 0 (#2649) + j = BasicJsonType(*opt); } else { diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index c8c9854ca..87e1dbc04 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -1950,12 +1950,42 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec assert_invariant(); } + private: + /// whether a basic_json specialization can be converted implicitly into this one; + /// with JSON_USE_IMPLICIT_CONVERSIONS set to 0, this is only the case if both share + /// the same string type (see https://github.com/nlohmann/json/issues/2649) + template + using is_implicitly_convertible_basic_json = std::integral_constant < bool, + (JSON_USE_IMPLICIT_CONVERSIONS != 0) + || std::is_same::value >; + + /// tag to select the constructor that performs the conversion from another basic_json specialization + struct convert_basic_json_tag {}; + + public: /// @brief create a JSON value from an existing one /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ template < typename BasicJsonType, detail::enable_if_t < - detail::is_basic_json::value&& !std::is_same::value, int > = 0 > + detail::is_basic_json::value&& !std::is_same::value + && is_implicitly_convertible_basic_json::value, int > = 0 > basic_json(const BasicJsonType& val) + : basic_json(val, convert_basic_json_tag{}) + {} + + /// @brief create a JSON value from an existing one + /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ + template < typename BasicJsonType, + detail::enable_if_t < + detail::is_basic_json::value&& !std::is_same::value + && !is_implicitly_convertible_basic_json::value, int > = 0 > + explicit basic_json(const BasicJsonType& val) + : basic_json(val, convert_basic_json_tag{}) + {} + + private: + template + basic_json(const BasicJsonType& val, convert_basic_json_tag /*unused*/) #if JSON_DIAGNOSTIC_POSITIONS : start_position(val.start_pos()), end_position(val.end_pos()) @@ -1975,6 +2005,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec assert_invariant(); } + public: /// @brief create a container (array or object) from an initializer list /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ basic_json(initializer_list_t init, @@ -2741,7 +2772,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec int > = 0 > BasicJsonType get_impl(detail::priority_tag<2> /*unused*/) const { - return *this; + return BasicJsonType(*this); } /*! @@ -2880,7 +2911,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec int> = 0> ValueType & get_to(ValueType& v) const { - v = *this; + v = ValueType(*this); return v; } diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 84ad40c74..a91bd9361 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -6960,7 +6960,9 @@ void to_json(BasicJsonType& j, const std::optional& opt) noexcept(std::is_not { if (opt.has_value()) { - j = *opt; + // explicit construction, as the conversion from a basic_json with a different + // string type is explicit if JSON_USE_IMPLICIT_CONVERSIONS is 0 (#2649) + j = BasicJsonType(*opt); } else { @@ -28726,12 +28728,42 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec assert_invariant(); } + private: + /// whether a basic_json specialization can be converted implicitly into this one; + /// with JSON_USE_IMPLICIT_CONVERSIONS set to 0, this is only the case if both share + /// the same string type (see https://github.com/nlohmann/json/issues/2649) + template + using is_implicitly_convertible_basic_json = std::integral_constant < bool, + (JSON_USE_IMPLICIT_CONVERSIONS != 0) + || std::is_same::value >; + + /// tag to select the constructor that performs the conversion from another basic_json specialization + struct convert_basic_json_tag {}; + + public: /// @brief create a JSON value from an existing one /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ template < typename BasicJsonType, detail::enable_if_t < - detail::is_basic_json::value&& !std::is_same::value, int > = 0 > + detail::is_basic_json::value&& !std::is_same::value + && is_implicitly_convertible_basic_json::value, int > = 0 > basic_json(const BasicJsonType& val) + : basic_json(val, convert_basic_json_tag{}) + {} + + /// @brief create a JSON value from an existing one + /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ + template < typename BasicJsonType, + detail::enable_if_t < + detail::is_basic_json::value&& !std::is_same::value + && !is_implicitly_convertible_basic_json::value, int > = 0 > + explicit basic_json(const BasicJsonType& val) + : basic_json(val, convert_basic_json_tag{}) + {} + + private: + template + basic_json(const BasicJsonType& val, convert_basic_json_tag /*unused*/) #if JSON_DIAGNOSTIC_POSITIONS : start_position(val.start_pos()), end_position(val.end_pos()) @@ -28751,6 +28783,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec assert_invariant(); } + public: /// @brief create a container (array or object) from an initializer list /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ basic_json(initializer_list_t init, @@ -29517,7 +29550,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec int > = 0 > BasicJsonType get_impl(detail::priority_tag<2> /*unused*/) const { - return *this; + return BasicJsonType(*this); } /*! @@ -29656,7 +29689,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec int> = 0> ValueType & get_to(ValueType& v) const { - v = *this; + v = ValueType(*this); return v; } diff --git a/tests/src/unit-alt-string.cpp b/tests/src/unit-alt-string.cpp index 8b86c78e8..e503a4914 100644 --- a/tests/src/unit-alt-string.cpp +++ b/tests/src/unit-alt-string.cpp @@ -13,6 +13,7 @@ #include #include +#include #include #include @@ -423,6 +424,41 @@ TEST_CASE("alternative string type") CHECK(j2.dump() == R"({"/foo/0":"bar","/foo/1":"baz"})"); } + SECTION("conversion between basic_json specializations (#2649)") + { + // explicit conversions are always possible + CHECK(std::is_constructible::value); + CHECK(std::is_constructible::value); + CHECK(std::is_constructible::value); + CHECK(std::is_constructible::value); + + // specializations with the same string type are implicitly convertible + CHECK(std::is_convertible::value); + CHECK(std::is_convertible::value); + + // specializations with different string types are only implicitly convertible + // if implicit conversions are enabled +#if JSON_USE_IMPLICIT_CONVERSIONS + CHECK(std::is_convertible::value); + CHECK(std::is_convertible::value); +#else + CHECK_FALSE(std::is_convertible::value); + CHECK_FALSE(std::is_convertible::value); +#endif + + // get() works in either case + const nlohmann::json j = {{"foo", 1}, {"bar", true}}; + CHECK(j.get() == nlohmann::ordered_json(j)); + // (only a number is converted here, as objects and strings are affected by #3425) + CHECK(nlohmann::json(42).get() == 42); + CHECK(alt_json(nlohmann::json(42)) == 42); + + // get_to() also works in either case + alt_json a; + nlohmann::json(42).get_to(a); + CHECK(a == 42); + } + SECTION("strict enum") { // regression test for #5667: NLOHMANN_JSON_SERIALIZE_ENUM_STRICT's from_json From b9850740b965b0c6ff126400f6ac52a0ec577ae1 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 4 Oct 2026 11:48:47 +0200 Subject: [PATCH 13/22] Add regression test for converting json to std::variant (#5595) * Add regression test for converting json to std::variant (#5066) With 3.10.5, get>() was well-formed through the string from_json overload, so the implicit conversion operator was a candidate when converting json to std::variant, and MSVC picked it over the variant's converting constructor. The tightened constraints from #3427 and #3604 (3.11.0) removed that path; this test guards against regressions. Signed-off-by: Niels Lohmann * Fix clang-tidy and clang 6 in the #5066 regression test ci_clang_tidy asked for emplace_back instead of push_back. The push_back is the point of the test: #5066 is about the implicit conversion from json to the vector's value type, which emplace_back would bypass. Silence the check on that line. clang 5 and 6 cannot instantiate std::variant from libstdc++ 10's ("cannot cast private base class"), which broke ci_test_compilers_clang (6). Tested with the CI images: clang 7 to 11 compile and pass, including clang 11 with libstdc++ 10. Skip the runtime check for clang before 7; the static_assert still runs. Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann --- tests/src/unit-regression2.cpp | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/src/unit-regression2.cpp b/tests/src/unit-regression2.cpp index 1ea5ac595..79ca5a770 100644 --- a/tests/src/unit-regression2.cpp +++ b/tests/src/unit-regression2.cpp @@ -808,6 +808,26 @@ TEST_CASE("regression tests 2") CHECK(j == k); } +#ifdef JSON_HAS_CPP_17 + SECTION("issue #5066 - MSVC converts json to std::variant via the conversion operator") + { + // std::variant must not be retrievable via get<>(), because otherwise the + // implicit conversion operator becomes a candidate that MSVC picks over the variant's + // converting constructor, routing a number through the string from_json overload + static_assert(!nlohmann::detail::is_detected>::value, + "std::variant must not be retrievable via get<>()"); + + // clang before 7 cannot instantiate libstdc++'s std::variant +#if !(defined(__clang__) && __clang_major__ < 7) + // push_back, not emplace_back: #5066 needs the implicit conversion + // from json to the vector's value type + std::vector> v; + v.push_back(json(1)); // NOLINT(hicpp-use-emplace,modernize-use-emplace) + CHECK(std::get<0>(v[0]) == 1); +#endif + } +#endif + SECTION("issue #3669 - invalid use of incomplete type with optional member and to_json") { const Issue3669Holder h{}; From 38a2db260c48efc4887613b412a8c941594988fa Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 4 Oct 2026 11:50:42 +0200 Subject: [PATCH 14/22] Remove dead metaprogramming and duplicated code in traits and pointers (#5728) * Remove unused is_sax and is_detected_convertible detail::is_sax had no user: the parser and the binary reader only use is_sax_static_asserts, so is_sax was a second, unchecked copy of the SAX event list. is_sax_static_asserts asserted boolean(bool) twice in a row, and detail::is_detected_convertible was never used anywhere. Remove all three and include for size_t instead of . Only names in nlohmann::detail are removed; behavior, public API and ABI are unchanged. The diagnostics for an incomplete SAX handler are the same, apart from the duplicated boolean() message. Part of #5708 Signed-off-by: Niels Lohmann * Replace meta/logic.hpp with a disjunction trait meta/logic.hpp added a second set of type-level boolean helpers (cxpr_and, cxpr_or, cxpr_not, ...) next to the existing conjunction and negation in type_traits.hpp. It was used only by one static_assert in from_json_tuple_impl, two of its templates were never used, and it was the only header without the license banner and relied on transitive includes for . Add the missing disjunction next to conjunction and negation, use the three in the static_assert, and delete logic.hpp together with its BUILD.bazel entry. same_sign now uses disjunction as well, which resolves the 2022 TODO waiting for such a trait. The static_assert accepts and rejects the same types as before. Only names in nlohmann::detail change; behavior, public API and ABI are unchanged. Part of #5708 Signed-off-by: Niels Lohmann * Remove unused would_call_std_* from NLOHMANN_CAN_CALL_STD_FUNC_IMPL Besides detail::result_of_begin/end, which is_range and iterator_t use, the macro defined a namespace detail2 with a tag type, a catch-all overload and would_call_std_begin/end, plus would_call_std_begin/end structs directly in namespace nlohmann. Nothing has used them since they were added in #3020. Reduce the macro to its detail part. Without the trailing struct the ';' after the two invocations would be an empty declaration that -Wextra-semi flags, so drop it. macro_scope.hpp included meta/detected.hpp only for this macro; all users of detected.hpp include it (or type_traits.hpp) themselves, so remove the include. Behavior and ABI are unchanged. The undocumented, untested and unused names nlohmann::would_call_std_begin, nlohmann::would_call_std_end and namespace nlohmann::detail2 are no longer declared. Part of #5708 Signed-off-by: Niels Lohmann * Simplify is_ordered_map to reuse has_capacity is_ordered_map re-detected capacity() with a C++03 sizeof/vararg trick right after has_capacity did the same detection through is_detected. For ordered_map, the old trick took the address of std::vector::capacity, which [namespace.std]/6 makes unspecified. Reuse has_capacity instead, which removes the unspecified-behavior pointer-to-std-member and two NOLINT suppressions. Part of #5708 Signed-off-by: Niels Lohmann * Remove duplicate const overload of json_pointer::get_checked The const and non-const get_checked() overloads had byte-identical 50-line bodies, differing only in the signature. The remaining template deduces a const-qualified BasicJsonType for const callers, so at(), the out_of_range::create() calls and the bounds check all still work. Part of #5708 Signed-off-by: Niels Lohmann * Fix tautological clause in iter_impl's iterator category assertion The static_assert meant to check the LegacyBidirectionalIterator named requirement had a first clause comparing std::bidirectional_iterator_tag to itself, which is always true and checks nothing; only array_t::iterator was actually being checked, despite the message claiming object iterators were checked too. Drop the tautological clause, reword the message to describe what is actually checked, and note that object_t may use a forward-only iterator as long as reverse iteration and operator-- are unused. The check is intentionally not extended to object_t::iterator, since that would reject object types with forward-only iterators that compile and work correctly today. Part of #5708 Signed-off-by: Niels Lohmann * Fix misplaced and stale comments in JSON_HAS_RANGES and conversions The JSON_HAS_RANGES feature-detection block had its libc++ comment sitting above the clang+libstdc++ branch it does not describe, leaving the libc++ branch uncommented and the clang+libstdc++ branch without its own rationale. Move each comment to sit under its own branch, and give the clang+libstdc++ branch (added in issue 5161) its own one-line reason referencing that issue instead of reusing the libc++ branch's comment. Also fix a duplicated-word typo ("in large in large cpp files") in from_json.hpp, drop two unanswered 2017 design questions left as comments in type_traits.hpp and from_json.hpp that no longer reflect open questions, and correct NLOHMANN_JSON_SERIALIZE_ENUM_STRICT's @since tag from 3.12.0 to 3.13.0, the release it was actually introduced in. Part of #5708 Signed-off-by: Niels Lohmann * Support any-rank C arrays in from_json, not just rank 1-4 from_json() for C arrays had four hand-unrolled overloads (rank 1-4, added incrementally in #4262), each with its own nested loops. to_json() already handles any rank recursively, so a rank-5+ C array could be serialized but not read back with get_to()/get<>(). Replace the four overloads with one from_json() SFINAE-constrained on get::type>() existing, forwarding to a pair of mutually recursive from_json_c_array_element() helpers: one assigns a non-array element via get(), the other loops over a array element and recurses one dimension at a time. Each dimension still goes through at(), so type_error.304/out_of_range.401 stay unchanged; ranks 1-4 keep their existing behavior and semantics. Adds rank-5 round-trip and mismatched-shape tests to unit-conversions.cpp. Public API: additive only (rank 5+ C arrays become readable). Signed-off-by: Niels Lohmann #5708 item 1 * Move templated_json_throw into nlohmann::detail templated_json_throw() was defined in macro_scope.hpp, which is included outside NLOHMANN_JSON_NAMESPACE_BEGIN, so the helper leaked into the global namespace as ::templated_json_throw with no ABI tag. Unqualified lookup in NLOHMANN_JSON_SERIALIZE_ENUM_STRICT could then bind to a same-named function declared in the user's own namespace instead, which fails to compile with Clang ("does not name a template"). Move the helper next to the exception classes in exceptions.hpp, inside nlohmann::detail, and call it qualified as ::nlohmann::detail::templated_json_throw<...>(...) from both macro expansion sites. Rewrite the doc comment to give the real reason for the helper (JSON_THROW may expand to code that discards its argument, e.g. when exceptions are disabled) and fix the "supress" typo. templated_json_throw was never released (added by #5151 after v3.12.0), so it can be moved freely. Adds a regression test that expands NLOHMANN_JSON_SERIALIZE_ENUM_STRICT inside a namespace declaring its own templated_json_throw. Public API: no change (::templated_json_throw was an unreleased, unintentional global-namespace leak with no callers relying on its location). Overlaps #5698, which rewrites the same two macro call lines; the overlapping hunks are small and should be trivial to reconcile on rebase. Signed-off-by: Niels Lohmann #5708 item 2 * Factor the repeated JSON_HAS_RANGES/MinGW guard into one macro The std::ranges view conversion (excluded on MinGW because of its incomplete C++20 ranges support, #4916) was gated by the same #if JSON_HAS_RANGES && !defined(__MINGW32__) condition at seven independent sites in to_json.hpp and type_traits.hpp, with the MinGW rationale duplicated in two of them and missing from the rest. Since the sites come in matching pairs (one enables is_compatible_range_view and a view-based overload, the other adds the exclusion to the plain-array-type overload), a drift between any pair would produce an ambiguous or missing overload on exactly one platform. Add JSON_HAS_RANGE_VIEW_CONVERSION next to JSON_HAS_RANGES in macro_scope.hpp, combining both conditions with the #4916 reasoning in one place, #undef it in macro_unscope.hpp, and use it at all seven sites. This does not fold the MinGW check into JSON_HAS_RANGES itself: JSON_HAS_RANGES is user-overridable and also gates the enable_borrowed_range specialization in iteration_proxy.hpp, which is not excluded on MinGW. No behavior or public API change: JSON_HAS_RANGE_VIEW_CONVERSION expands to exactly the condition that was previously written out at each site. Overlaps #5585, #5600 and #3575, which touch the same to_json.hpp and type_traits.hpp lines; the change here is a mechanical search-and-replace of the guard condition and should rebase cleanly. Signed-off-by: Niels Lohmann #5708 item 11 * De-duplicate from_json.hpp's map and array-fallback bodies Several from_json() overload pairs in from_json.hpp were copies of each other, so a fix has to be applied twice (as #5681 already does): - from_json(..., std::map&) and from_json(..., std::unordered_map&) for non-string keys had identical 16-line bodies: array check, m.clear(), pair check loop, m.emplace(...). Route both through a new from_json_pair_array_to_map(j, m) helper. - The from_json_array_impl priority_tag<1> and priority_tag<0> fallbacks ran the same std::transform/std::inserter loop, differing only in ret.reserve(j.size()). Merge them into one body and, modeled on the existing from_json_object_reserve, add a from_json_array_reserve pair so the reserve() call is only made for ConstructibleArrayType that support it. Error ids (type_error.302), messages, diagnostic paths ((at(0)/at(1)) and behavior for types with/without reserve() are unchanged; only the duplication is removed. Public API: no change. Overlaps #5681, which changes the "&j" to "&p" line in both map bodies; the shared helper here should make that a one-line change instead of two on rebase. Signed-off-by: Niels Lohmann #5708 item 5 * Unify json_pointer's three array-index parsers array_index(), contains() and get_checked_or_null() each re-implemented the RFC 6901 array-index rules and the size_type range check: array_index() does the canonical parse and throws; contains() (which must not throw, #5395) re-validates every digit by hand and runs its own strtoull/ERANGE check before calling array_index() anyway, parsing every array token twice; get_checked_or_null() wraps array_index() in JSON_TRY/ JSON_INTERNAL_CATCH (detail::out_of_range&) to turn an unrepresentable index into "not found". Add a single private, noexcept parse_array_index(s, idx) returning an array_index_status (ok / leading_zero / not_a_number / unresolved / exceeds_size_type). array_index() becomes a thin wrapper mapping each status to the existing parse_error.106/109 or out_of_range.404/410; contains() and get_checked_or_null() switch on the status directly. This removes contains()'s digit-validation loop and its second strtoull call, and get_checked_or_null()'s JSON_TRY/JSON_INTERNAL_CATCH. Bugfix as a consequence: get_checked_or_null()'s JSON_TRY/ JSON_INTERNAL_CATCH was dead code under JSON_NOEXCEPTION (JSON_TRY expands to "if(true)" and the catch to "if(false)", so JSON_THROW's std::abort() ran unconditionally), meaning value() and contains() would abort instead of returning the default/false for an out-of-range-sized or oversized array index when exceptions are disabled (#5672). Switching on parse_array_index()'s return value instead of relying on an actual throw/catch fixes this: get_checked_or_null() now returns nullptr for array_index_status::unresolved/exceeds_size_type in every build configuration, and still calls JSON_THROW (aborting under JSON_NOEXCEPTION, as before) only for a malformed index (leading_zero/not_a_number), matching its documented @throw list. All existing error ids, messages and diagnostic paths are unchanged; a few reference tokens that used to fail contains()'s manual per-character validation (e.g. "1a") now fail via array_index_status::unresolved instead, with no observable difference since contains() only returns bool. Adds regression tests to unit-element_access2.cpp's "access on array type" section covering value() with an index that exceeds size_type and one with a trailing non-digit, both of which must yield the default value rather than abort/throw. Public API: no change. Overlaps #5700, #5614 and #5692, which touch the contains() and get_checked_or_null() array hunks; this change replaces those hunks with calls into the new shared parser, so a rebase will need to re-apply their token-handling changes (e.g. the empty-token case) on top of the switch statements here. Signed-off-by: Niels Lohmann #5708 item 4 * Regenerate single_include after merging develop The merge commit kept develop's single_include/nlohmann/json.hpp because make amalgamate saw it as up to date. Signed-off-by: Niels Lohmann * Address review: switch in array_index, drop redundant inline - json_pointer::array_index() dispatches on array_index_status with a switch, matching the other parse_array_index() caller - drop `inline` from the function templates this PR adds or moves in from_json.hpp - reword a comment that described the change rather than the code Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann --- BUILD.bazel | 1 - .../nlohmann/detail/conversions/from_json.hpp | 144 +-- .../nlohmann/detail/conversions/to_json.hpp | 10 +- include/nlohmann/detail/exceptions.hpp | 21 + .../nlohmann/detail/iterators/iter_impl.hpp | 8 +- include/nlohmann/detail/json_pointer.hpp | 252 +++--- include/nlohmann/detail/macro_scope.hpp | 60 +- include/nlohmann/detail/macro_unscope.hpp | 1 + .../nlohmann/detail/meta/call_std/begin.hpp | 2 +- include/nlohmann/detail/meta/call_std/end.hpp | 2 +- include/nlohmann/detail/meta/is_sax.hpp | 35 +- include/nlohmann/detail/meta/logic.hpp | 54 -- include/nlohmann/detail/meta/type_traits.hpp | 35 +- single_include/nlohmann/json.hpp | 819 +++++++----------- tests/src/unit-conversions.cpp | 59 ++ tests/src/unit-type_traits.cpp | 12 + 16 files changed, 597 insertions(+), 918 deletions(-) delete mode 100644 include/nlohmann/detail/meta/logic.hpp diff --git a/BUILD.bazel b/BUILD.bazel index 03f73fa92..a9b09fd79 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -53,7 +53,6 @@ cc_library( "include/nlohmann/detail/meta/detected.hpp", "include/nlohmann/detail/meta/identity_tag.hpp", "include/nlohmann/detail/meta/is_sax.hpp", - "include/nlohmann/detail/meta/logic.hpp", "include/nlohmann/detail/meta/std_fs.hpp", "include/nlohmann/detail/meta/type_traits.hpp", "include/nlohmann/detail/meta/void_t.hpp", diff --git a/include/nlohmann/detail/conversions/from_json.hpp b/include/nlohmann/detail/conversions/from_json.hpp index d5c3328ca..5274b9857 100644 --- a/include/nlohmann/detail/conversions/from_json.hpp +++ b/include/nlohmann/detail/conversions/from_json.hpp @@ -27,7 +27,6 @@ #include #include #include -#include #include #include @@ -211,62 +210,29 @@ inline void from_json(const BasicJsonType& j, std::valarray& l) }); } +// element is not itself a C array: read it directly +template +auto from_json_c_array_element(const BasicJsonType& j, T& e) +-> decltype(e = j.template get(), void()) +{ + e = j.template get(); +} + +// element is itself a C array: recurse one dimension at a time, so any rank is supported template -auto from_json(const BasicJsonType& j, T (&arr)[N]) // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) --> decltype(j.template get(), void()) +void from_json_c_array_element(const BasicJsonType& j, T (&arr)[N]) // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) { for (std::size_t i = 0; i < N; ++i) { - arr[i] = j.at(i).template get(); + from_json_c_array_element(j.at(i), arr[i]); } } -template -auto from_json(const BasicJsonType& j, T (&arr)[N1][N2]) // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) --> decltype(j.template get(), void()) +template +auto from_json(const BasicJsonType& j, T (&arr)[N]) // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) +-> decltype(j.template get::type>(), void()) { - for (std::size_t i1 = 0; i1 < N1; ++i1) - { - for (std::size_t i2 = 0; i2 < N2; ++i2) - { - arr[i1][i2] = j.at(i1).at(i2).template get(); - } - } -} - -template -auto from_json(const BasicJsonType& j, T (&arr)[N1][N2][N3]) // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) --> decltype(j.template get(), void()) -{ - for (std::size_t i1 = 0; i1 < N1; ++i1) - { - for (std::size_t i2 = 0; i2 < N2; ++i2) - { - for (std::size_t i3 = 0; i3 < N3; ++i3) - { - arr[i1][i2][i3] = j.at(i1).at(i2).at(i3).template get(); - } - } - } -} - -template -auto from_json(const BasicJsonType& j, T (&arr)[N1][N2][N3][N4]) // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) --> decltype(j.template get(), void()) -{ - for (std::size_t i1 = 0; i1 < N1; ++i1) - { - for (std::size_t i2 = 0; i2 < N2; ++i2) - { - for (std::size_t i3 = 0; i3 < N3; ++i3) - { - for (std::size_t i4 = 0; i4 < N4; ++i4) - { - arr[i1][i2][i3][i4] = j.at(i1).at(i2).at(i3).at(i4).template get(); - } - } - } - } + from_json_c_array_element(j, arr); } template @@ -286,20 +252,33 @@ auto from_json_array_impl(const BasicJsonType& j, std::array& arr, } } +// reserve() is called through this pair (modeled on from_json_object_reserve) +// so from_json_array_impl below has a single body for both ConstructibleArrayType +// that support reserve() and those that don't. +template +auto from_json_array_reserve(ConstructibleArrayType& arr, typename ConstructibleArrayType::size_type size, priority_tag<1> /*unused*/) +-> decltype(arr.reserve(size), void()) +{ + arr.reserve(size); +} + +template +void from_json_array_reserve(ConstructibleArrayType& /*arr*/, std::size_t /*size*/, priority_tag<0> /*unused*/) +{} + template::value, int> = 0> auto from_json_array_impl(const BasicJsonType& j, ConstructibleArrayType& arr, priority_tag<1> /*unused*/) -> decltype( - arr.reserve(std::declval()), j.template get(), void()) { using std::end; ConstructibleArrayType ret; - ret.reserve(j.size()); + from_json_array_reserve(ret, j.size(), priority_tag<1> {}); std::transform(j.begin(), j.end(), std::inserter(ret, end(ret)), [](const BasicJsonType & i) { @@ -310,27 +289,6 @@ auto from_json_array_impl(const BasicJsonType& j, ConstructibleArrayType& arr, p arr = std::move(ret); } -template::value, - int> = 0> -inline void from_json_array_impl(const BasicJsonType& j, ConstructibleArrayType& arr, - priority_tag<0> /*unused*/) -{ - using std::end; - - ConstructibleArrayType ret; - std::transform( - j.begin(), j.end(), std::inserter(ret, end(ret)), - [](const BasicJsonType & i) - { - // get() returns *this, this won't call a from_json - // method when value_type is BasicJsonType - return i.template get(); - }); - arr = std::move(ret); -} - template < typename BasicJsonType, typename ConstructibleArrayType, enable_if_t < is_constructible_array_type::value&& @@ -433,9 +391,7 @@ inline void from_json(const BasicJsonType& j, ConstructibleObjectType& obj) } // overload for arithmetic types, not chosen for basic_json template arguments -// (BooleanType, etc.); note: Is it really necessary to provide explicit -// overloads for boolean_t etc. in case of a custom BooleanType which is not -// an arithmetic type? +// (BooleanType, etc.) template < typename BasicJsonType, typename ArithmeticType, enable_if_t < std::is_arithmetic::value&& @@ -531,7 +487,7 @@ inline void from_json_tuple_impl(const BasicJsonType& j, std::pair& p, p template std::tuple from_json_tuple_impl(const BasicJsonType& j, identity_tag> /*unused*/, priority_tag<2> /*unused*/) { - static_assert(cxpr_and>, is_compatible_reference_type>...>::value, + static_assert(conjunction>, is_compatible_reference_type>...>::value, "Can not return a tuple containing references to types not contained in a Json, try Json::get_to()"); return from_json_tuple_impl_base<1, Args...>(j, index_sequence_for {}); } @@ -554,10 +510,10 @@ auto from_json(const BasicJsonType& j, TupleRelated&& t) return from_json_tuple_impl(j, std::forward(t), priority_tag<3> {}); } -template < typename BasicJsonType, typename Key, typename Value, typename Compare, typename Allocator, - typename = enable_if_t < !std::is_constructible < - typename BasicJsonType::string_t, Key >::value >> -inline void from_json(const BasicJsonType& j, std::map& m) +// shared body for std::map/std::unordered_map with a non-string Key: both +// containers are read from an array of [key, value] pairs the same way +template +void from_json_pair_array_to_map(const BasicJsonType& j, MapType& m) { if (JSON_HEDLEY_UNLIKELY(!j.is_array())) { @@ -570,33 +526,29 @@ inline void from_json(const BasicJsonType& j, std::map(), p.at(1).template get()); + m.emplace(p.at(0).template get(), p.at(1).template get()); } } +template < typename BasicJsonType, typename Key, typename Value, typename Compare, typename Allocator, + typename = enable_if_t < !std::is_constructible < + typename BasicJsonType::string_t, Key >::value >> +void from_json(const BasicJsonType& j, std::map& m) +{ + from_json_pair_array_to_map(j, m); +} + template < typename BasicJsonType, typename Key, typename Value, typename Hash, typename KeyEqual, typename Allocator, typename = enable_if_t < !std::is_constructible < typename BasicJsonType::string_t, Key >::value >> -inline void from_json(const BasicJsonType& j, std::unordered_map& m) +void from_json(const BasicJsonType& j, std::unordered_map& m) { - if (JSON_HEDLEY_UNLIKELY(!j.is_array())) - { - JSON_THROW(type_error::create(302, concat("type must be array, but is ", j.type_name()), &j)); - } - m.clear(); - for (const auto& p : j) - { - if (JSON_HEDLEY_UNLIKELY(!p.is_array())) - { - JSON_THROW(type_error::create(302, concat("type must be array, but is ", p.type_name()), &p)); - } - m.emplace(p.at(0).template get(), p.at(1).template get()); - } + from_json_pair_array_to_map(j, m); } #if JSON_HAS_FILESYSTEM || JSON_HAS_EXPERIMENTAL_FILESYSTEM -// Workaround for MSVC 19.51 (and possibly later): in large in large cpp files, the compiler may fail to resolve with generic has_from_json (issue #4996) +// Workaround for MSVC 19.51 (and possibly later): in large cpp files, the compiler may fail to resolve with generic has_from_json (issue #4996) template struct has_from_json : std::true_type {}; diff --git a/include/nlohmann/detail/conversions/to_json.hpp b/include/nlohmann/detail/conversions/to_json.hpp index 7b97068f3..2dba6c163 100644 --- a/include/nlohmann/detail/conversions/to_json.hpp +++ b/include/nlohmann/detail/conversions/to_json.hpp @@ -178,7 +178,7 @@ struct external_constructor template < typename BasicJsonType, typename CompatibleArrayType, enable_if_t < !std::is_same::value -#if JSON_HAS_RANGES && !defined(__MINGW32__) +#if JSON_HAS_RANGE_VIEW_CONVERSION && !is_compatible_range_view::value #endif , int > = 0 > @@ -222,9 +222,7 @@ struct external_constructor j.assert_invariant(); } - // std::ranges does not work properly on MinGW due to incomplete C++20 support - // see https://github.com/nlohmann/json/issues/4916 -#if JSON_HAS_RANGES && !defined(__MINGW32__) +#if JSON_HAS_RANGE_VIEW_CONVERSION template>::value, int> = 0> static void construct(BasicJsonType& j, CompatibleArrayType && arr) @@ -384,7 +382,7 @@ template < typename BasicJsonType, typename CompatibleArrayType, !std::is_same::value&& !is_compatible_binary_type::value&& !is_basic_json::value -#if JSON_HAS_RANGES && !defined(__MINGW32__) +#if JSON_HAS_RANGE_VIEW_CONVERSION && !is_compatible_range_view::value #endif , @@ -394,7 +392,7 @@ inline void to_json(BasicJsonType& j, const CompatibleArrayType& arr) external_constructor::construct(j, arr); } -#if JSON_HAS_RANGES && !defined(__MINGW32__) +#if JSON_HAS_RANGE_VIEW_CONVERSION template < typename BasicJsonType, typename T, enable_if_t < is_compatible_range_view>::value && !is_compatible_string_type>::value diff --git a/include/nlohmann/detail/exceptions.hpp b/include/nlohmann/detail/exceptions.hpp index 808a3b6bf..155f4a4e5 100644 --- a/include/nlohmann/detail/exceptions.hpp +++ b/include/nlohmann/detail/exceptions.hpp @@ -286,6 +286,27 @@ class other_error : public exception other_error(int id_, const char* what_arg) : exception(id_, what_arg) {} }; +/*! +@brief helper function to call JSON_THROW from a template +@note JSON_THROW is a macro that, depending on the JSON_THROW_USER / + JSON_TRY_USER / JSON_NOEXCEPTION configuration, may expand to code + that does not reference its argument (e.g. `std::abort()`), which + would trigger a compilation error if the argument's type depends on + a template parameter that is otherwise unused. Wrapping the call in + a templated function avoids this and gives the compiler a single + place to see the (possibly unused) parameter. +*/ +template +void templated_json_throw(ExceptionType exception) +{ + JSON_THROW(exception); + + // JSON_THROW may expand to code that discards its argument (e.g. when + // exceptions are disabled) - the cast below avoids an unused-parameter + // warning with -Werror in that case + (void)exception; +} + } // namespace detail NLOHMANN_JSON_NAMESPACE_END diff --git a/include/nlohmann/detail/iterators/iter_impl.hpp b/include/nlohmann/detail/iterators/iter_impl.hpp index 2115b6aeb..84fdd07f8 100644 --- a/include/nlohmann/detail/iterators/iter_impl.hpp +++ b/include/nlohmann/detail/iterators/iter_impl.hpp @@ -60,9 +60,11 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci static_assert(is_basic_json::type>::value, "iter_impl only accepts (const) basic_json"); // superficial check for the LegacyBidirectionalIterator named requirement - static_assert(std::is_base_of::value - && std::is_base_of::iterator_category>::value, - "basic_json iterator assumes array and object type iterators satisfy the LegacyBidirectionalIterator named requirement."); + // note: only array_t::iterator is checked here; object_t::iterator may be + // a forward-only iterator as long as reverse iteration and operator-- + // are never used on it + static_assert(std::is_base_of::iterator_category>::value, + "basic_json iterator assumes array type iterators satisfy the LegacyBidirectionalIterator named requirement."); public: /// The std::iterator class template (used as a base class to provide typedefs) is deprecated in C++17. diff --git a/include/nlohmann/detail/json_pointer.hpp b/include/nlohmann/detail/json_pointer.hpp index a10f4f39f..788af1047 100644 --- a/include/nlohmann/detail/json_pointer.hpp +++ b/include/nlohmann/detail/json_pointer.hpp @@ -240,6 +240,72 @@ class json_pointer } private: + /*! + @brief result of @ref parse_array_index + + @ref array_index maps each value to the corresponding parse_error/out_of_range + exception; @ref contains and @ref get_checked_or_null, which must not throw for + an out-of-range or unrepresentable index, switch on it directly instead. + */ + enum class array_index_status + { + ok, ///< @a s is a valid, representable array index + leading_zero, ///< @a s begins with '0' but has more than one character + not_a_number, ///< @a s does not begin with a digit + unresolved, ///< @a s could not be converted to an integer + exceeds_size_type ///< @a s converts to an integer that exceeds size_type + }; + + /*! + @param[in] s reference token to be converted into an array index + @param[out] idx the integer representation of @a s if @ref array_index_status::ok + is returned; left unchanged otherwise + + @return whether @a s is a valid array index, and if not, why + + @note this function never throws; @ref array_index and the callers that must not + throw (@ref contains, @ref get_checked_or_null) build on it instead of each + re-implementing the RFC 6901 digit rules and the @a size_type range check + */ + template + static array_index_status parse_array_index(const string_t& s, typename BasicJsonType::size_type& idx) noexcept + { + using size_type = typename BasicJsonType::size_type; + + // error condition (cf. RFC 6901, Sect. 4) + if (JSON_HEDLEY_UNLIKELY(s.size() > 1 && s[0] == '0')) + { + return array_index_status::leading_zero; + } + + // error condition (cf. RFC 6901, Sect. 4) + if (JSON_HEDLEY_UNLIKELY(s.size() > 1 && !(s[0] >= '1' && s[0] <= '9'))) + { + return array_index_status::not_a_number; + } + + const char* p = s.data(); + char* p_end = nullptr; // NOLINT(misc-const-correctness) + errno = 0; // strtoull doesn't reset errno + const unsigned long long res = std::strtoull(p, &p_end, 10); // NOLINT(runtime/int) + if (p == p_end // invalid input or empty string + || errno == ERANGE // out of range + || JSON_HEDLEY_UNLIKELY(static_cast(p_end - p) != s.size())) // incomplete read + { + return array_index_status::unresolved; + } + + // the index does not fit into size_type; on 64-bit platforms this is + // only SIZE_MAX itself (see #2203 and #5395) + if (res >= static_cast((std::numeric_limits::max)())) // NOLINT(runtime/int) + { + return array_index_status::exceeds_size_type; + } + + idx = static_cast(res); + return array_index_status::ok; + } + /*! @param[in] s reference token to be converted into an array index @@ -253,39 +319,23 @@ class json_pointer template static typename BasicJsonType::size_type array_index(const string_t& s) { - using size_type = typename BasicJsonType::size_type; - - // error condition (cf. RFC 6901, Sect. 4) - if (JSON_HEDLEY_UNLIKELY(s.size() > 1 && s[0] == '0')) + typename BasicJsonType::size_type idx{}; + switch (parse_array_index(s, idx)) { - JSON_THROW(detail::parse_error::create(106, 0, detail::concat("array index '", s, "' must not begin with '0'"), nullptr)); + case array_index_status::leading_zero: + JSON_THROW(detail::parse_error::create(106, 0, detail::concat("array index '", s, "' must not begin with '0'"), nullptr)); + case array_index_status::not_a_number: + JSON_THROW(detail::parse_error::create(109, 0, detail::concat("array index '", s, "' is not a number"), nullptr)); + case array_index_status::unresolved: + JSON_THROW(detail::out_of_range::create(404, detail::concat("unresolved reference token '", s, "'"), nullptr)); + case array_index_status::exceeds_size_type: + JSON_THROW(detail::out_of_range::create(410, detail::concat("array index ", s, " exceeds size_type"), nullptr)); + case array_index_status::ok: + default: + break; } - // error condition (cf. RFC 6901, Sect. 4) - if (JSON_HEDLEY_UNLIKELY(s.size() > 1 && !(s[0] >= '1' && s[0] <= '9'))) - { - JSON_THROW(detail::parse_error::create(109, 0, detail::concat("array index '", s, "' is not a number"), nullptr)); - } - - const char* p = s.data(); - char* p_end = nullptr; // NOLINT(misc-const-correctness) - errno = 0; // strtoull doesn't reset errno - const unsigned long long res = std::strtoull(p, &p_end, 10); // NOLINT(runtime/int) - if (p == p_end // invalid input or empty string - || errno == ERANGE // out of range - || JSON_HEDLEY_UNLIKELY(static_cast(p_end - p) != s.size())) // incomplete read - { - JSON_THROW(detail::out_of_range::create(404, detail::concat("unresolved reference token '", s, "'"), nullptr)); - } - - // the index does not fit into size_type; on 64-bit platforms this is - // only SIZE_MAX itself (see #2203 and #5395) - if (res >= static_cast((std::numeric_limits::max)())) // NOLINT(runtime/int) - { - JSON_THROW(detail::out_of_range::create(410, detail::concat("array index ", s, " exceeds size_type"), nullptr)); - } - - return static_cast(res); + return idx; } JSON_PRIVATE_UNLESS_TESTED: @@ -590,63 +640,6 @@ class json_pointer return *ptr; } - /*! - @throw parse_error.106 if an array index begins with '0' - @throw parse_error.109 if an array index was not a number - @throw out_of_range.402 if the array index '-' is used - @throw out_of_range.404 if the JSON pointer can not be resolved - */ - template - const BasicJsonType& get_checked(const BasicJsonType* ptr) const - { - for (const auto& reference_token : reference_tokens) - { - switch (ptr->type()) - { - case detail::value_t::object: - { - // note: at performs range check - ptr = &ptr->at(reference_token); - break; - } - - case detail::value_t::array: - { - if (JSON_HEDLEY_UNLIKELY(reference_token == "-")) - { - // "-" always fails the range check - JSON_THROW(detail::out_of_range::create(402, detail::concat( - "array index '-' (", std::to_string(ptr->m_data.m_value.array->size()), - ") is out of range"), ptr)); - } - - const auto idx = array_index(reference_token); - // Bounds check before access to avoid exception with JSON_NOEXCEPTION - if (JSON_HEDLEY_UNLIKELY(idx >= ptr->m_data.m_value.array->size())) - { - JSON_THROW(detail::out_of_range::create(401, detail::concat( - "array index ", std::to_string(idx), " is out of range"), ptr)); - } - ptr = &ptr->operator[](idx); - break; - } - - case detail::value_t::null: - case detail::value_t::string: - case detail::value_t::boolean: - case detail::value_t::number_integer: - case detail::value_t::number_unsigned: - case detail::value_t::number_float: - case detail::value_t::binary: - case detail::value_t::discarded: - default: - JSON_THROW(detail::out_of_range::create(404, detail::concat("unresolved reference token '", reference_token, "'"), ptr)); - } - } - - return *ptr; - } - /*! @brief return a pointer to the pointed to value, or `nullptr` if the pointer cannot be resolved because a key is missing, an array @@ -685,29 +678,24 @@ class json_pointer return nullptr; } - // tokens that array_index() rejects with parse_error.106/109 - // are passed on to it; all other tokens that it would reject - // with out_of_range.404/410 are detected here, so that this - // also works without exceptions - if (JSON_HEDLEY_UNLIKELY(reference_token.size() > 1 && !(reference_token[0] >= '1' && reference_token[0] <= '9'))) + // a malformed index throws parse_error.106/109; an + // index that is syntactically valid but cannot be + // represented (out_of_range.404/410) is treated like an + // out-of-range index below + typename BasicJsonType::size_type idx{}; + switch (parse_array_index(reference_token, idx)) { - static_cast(array_index(reference_token)); // throws parse_error.106/109 + case array_index_status::leading_zero: + JSON_THROW(detail::parse_error::create(106, 0, detail::concat("array index '", reference_token, "' must not begin with '0'"), nullptr)); + case array_index_status::not_a_number: + JSON_THROW(detail::parse_error::create(109, 0, detail::concat("array index '", reference_token, "' is not a number"), nullptr)); + case array_index_status::unresolved: + case array_index_status::exceeds_size_type: + return nullptr; + case array_index_status::ok: + default: + break; } - if (JSON_HEDLEY_UNLIKELY(reference_token.empty() || !std::all_of(reference_token.begin(), reference_token.end(), [](const char c) - { - return c >= '0' && c <= '9'; - }))) - { - return nullptr; - } - errno = 0; // strtoull() does not reset errno on success - char* p_end = nullptr; // NOLINT(misc-const-correctness) - const unsigned long long magnitude = std::strtoull(reference_token.data(), &p_end, 10); // NOLINT(runtime/int) - if (JSON_HEDLEY_UNLIKELY(errno == ERANGE || magnitude >= static_cast((std::numeric_limits::max)()))) // NOLINT(runtime/int) - { - return nullptr; - } - const auto idx = static_cast(magnitude); if (JSON_HEDLEY_UNLIKELY(idx >= ptr->m_data.m_value.array->size())) { @@ -734,8 +722,8 @@ class json_pointer } /*! - @throw parse_error.106 if an array index begins with '0' - @throw parse_error.109 if an array index was not a number + @note unlike array_index(), this never throws: a malformed or unrepresentable + array index reference token is treated like a missing key (see #5395) */ template bool contains(const BasicJsonType* ptr) const @@ -763,49 +751,17 @@ class json_pointer // "-" always fails the range check return false; } - if (JSON_HEDLEY_UNLIKELY(reference_token.empty())) - { - // an empty reference token is not an array index; array_index() - // would throw out_of_range.404 -- contains() must not throw (see #5395) - return false; - } - if (JSON_HEDLEY_UNLIKELY(reference_token.size() == 1 && !('0' <= reference_token[0] && reference_token[0] <= '9'))) - { - // invalid char - return false; - } - if (JSON_HEDLEY_UNLIKELY(reference_token.size() > 1)) - { - if (JSON_HEDLEY_UNLIKELY(!('1' <= reference_token[0] && reference_token[0] <= '9'))) - { - // the first char should be between '1' and '9' - return false; - } - for (std::size_t i = 1; i < reference_token.size(); i++) - { - if (JSON_HEDLEY_UNLIKELY(!('0' <= reference_token[i] && reference_token[i] <= '9'))) - { - // other char should be between '0' and '9' - return false; - } - } - } - // the reference token consists only of digits at this point (cf. checks - // above); however, its numeric value might not be representable, in which - // case array_index() would throw out_of_range.404/410 -- contains() must - // not throw (see #5395), so such a reference token is treated as "not found" - errno = 0; // strtoull() does not reset errno on success - char* p_end = nullptr; // NOLINT(misc-const-correctness) - const unsigned long long magnitude = std::strtoull(reference_token.data(), &p_end, 10); // NOLINT(runtime/int) - if (JSON_HEDLEY_UNLIKELY(errno == ERANGE // the value exceeds ULLONG_MAX - || magnitude >= static_cast((std::numeric_limits::max)()))) // NOLINT(runtime/int) + // any parse failure (malformed index, or one that is syntactically + // valid but not representable as size_type) means the reference + // token cannot denote an existing array element -- contains() must + // not throw (see #5395), so it is treated as "not found" + typename BasicJsonType::size_type idx{}; + if (JSON_HEDLEY_UNLIKELY(parse_array_index(reference_token, idx) != array_index_status::ok)) { - // the array index cannot be represented as size_type return false; } - const auto idx = array_index(reference_token); if (idx >= ptr->size()) { // index out of range diff --git a/include/nlohmann/detail/macro_scope.hpp b/include/nlohmann/detail/macro_scope.hpp index 3a5f7e606..17118927c 100644 --- a/include/nlohmann/detail/macro_scope.hpp +++ b/include/nlohmann/detail/macro_scope.hpp @@ -9,7 +9,6 @@ #pragma once #include // declval, pair -#include #include // This file contains all internal macro definitions (except those affecting ABI) @@ -140,10 +139,12 @@ // libstdc++ < 11 has incomplete C++20 ranges (issue #4440) #elif defined(_GLIBCXX_RELEASE) && _GLIBCXX_RELEASE < 11 #define JSON_HAS_RANGES 0 - // libc++ < 16 has incomplete C++20 ranges (issue #4440) + // clang < 16 with libstdc++ does not implement the ranges customization + // points libstdc++ declares, so its C++20 ranges support is incomplete (issue #5161) #elif defined(__clang__) && !defined(__apple_build_version__) \ && __clang_major__ < 16 && defined(__GLIBCXX__) #define JSON_HAS_RANGES 0 + // libc++ < 16 has incomplete C++20 ranges (issue #4440) #elif defined(_LIBCPP_VERSION) && _LIBCPP_VERSION < 160000 #define JSON_HAS_RANGES 0 // nvcc CUDA 12.0/12.1 chokes on the enable_borrowed_range variable-template @@ -158,6 +159,18 @@ #endif #endif +// std::ranges view conversion (to_json/is_compatible_array_type_impl) additionally +// needs to be disabled on MinGW, whose std::ranges support is incomplete +// (issue #4916); this macro combines both conditions so the check and its +// reason are not duplicated at every use site. +#ifndef JSON_HAS_RANGE_VIEW_CONVERSION + #if JSON_HAS_RANGES && !defined(__MINGW32__) + #define JSON_HAS_RANGE_VIEW_CONVERSION 1 + #else + #define JSON_HAS_RANGE_VIEW_CONVERSION 0 + #endif +#endif + #ifndef JSON_HAS_STD_FORMAT #if defined(JSON_HAS_CPP_20) && defined(__cpp_lib_format) #define JSON_HAS_STD_FORMAT 1 @@ -279,21 +292,6 @@ -/*! -@brief function to wrap JSON_THROW_MACRO - there can be compilation errors about - there being no arguments to JSON_THROW that depend on template arguments - if this is not used to call JSON_THROW -*/ -template -void templated_json_throw(ExceptionType exception) -{ - JSON_THROW(exception); - - /* JSON_THROW(exception) discards exception and aborts - void cast needed to supress - compilation error if compiled with -Werror and Wunused-parameter */ - (void)exception; -} - /*! @brief macro to briefly define a mapping between an enum and JSON with exception on invalid input @@ -314,7 +312,7 @@ void templated_json_throw(ExceptionType exception) return ej_pair.first == e; \ }); \ if (it != std::end(m)) j = it->second; \ - else templated_json_throw(nlohmann::detail::out_of_range::create(410,"enum value out of range for " #ENUM_TYPE, nullptr)); \ + else ::nlohmann::detail::templated_json_throw(nlohmann::detail::out_of_range::create(410,"enum value out of range for " #ENUM_TYPE, nullptr)); \ } \ template \ inline void from_json(const BasicJsonType& j, ENUM_TYPE& e) \ @@ -329,7 +327,7 @@ void templated_json_throw(ExceptionType exception) return ej_pair.second == j; \ }); \ if (it != std::end(m)) e = it->first; \ - else templated_json_throw(nlohmann::detail::out_of_range::create(410, nlohmann::detail::concat("enum value out of range for " #ENUM_TYPE ": ", j.dump(-1, ' ', false, nlohmann::detail::error_handler_t::replace)), &j)); \ + else ::nlohmann::detail::templated_json_throw(nlohmann::detail::out_of_range::create(410, nlohmann::detail::concat("enum value out of range for " #ENUM_TYPE ": ", j.dump(-1, ' ', false, nlohmann::detail::error_handler_t::replace)), &j)); \ } // Ugly macros to avoid uglier copy-paste when specializing basic_json. They @@ -874,30 +872,6 @@ void templated_json_throw(ExceptionType exception) \ template \ using result_of_##std_name = decltype(std_name(std::declval()...)); \ - } \ - \ - namespace detail2 { \ - struct std_name##_tag \ - { \ - }; \ - \ - template \ - std_name##_tag std_name(T&&...); \ - \ - template \ - using result_of_##std_name = decltype(std_name(std::declval()...)); \ - \ - template \ - struct would_call_std_##std_name \ - { \ - static constexpr auto const value = ::nlohmann::detail:: \ - is_detected_exact::value; \ - }; \ - } /* namespace detail2 */ \ - \ - template \ - struct would_call_std_##std_name : detail2::would_call_std_##std_name \ - { \ } #ifndef JSON_USE_IMPLICIT_CONVERSIONS diff --git a/include/nlohmann/detail/macro_unscope.hpp b/include/nlohmann/detail/macro_unscope.hpp index 1e6e6cce6..8e1d49842 100644 --- a/include/nlohmann/detail/macro_unscope.hpp +++ b/include/nlohmann/detail/macro_unscope.hpp @@ -35,6 +35,7 @@ #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM #undef JSON_HAS_THREE_WAY_COMPARISON #undef JSON_HAS_RANGES + #undef JSON_HAS_RANGE_VIEW_CONVERSION #undef JSON_HAS_STD_FORMAT #undef JSON_HAS_STATIC_RTTI #undef JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON diff --git a/include/nlohmann/detail/meta/call_std/begin.hpp b/include/nlohmann/detail/meta/call_std/begin.hpp index a086a5f3d..d12bbf7da 100644 --- a/include/nlohmann/detail/meta/call_std/begin.hpp +++ b/include/nlohmann/detail/meta/call_std/begin.hpp @@ -12,6 +12,6 @@ NLOHMANN_JSON_NAMESPACE_BEGIN -NLOHMANN_CAN_CALL_STD_FUNC_IMPL(begin); +NLOHMANN_CAN_CALL_STD_FUNC_IMPL(begin) NLOHMANN_JSON_NAMESPACE_END diff --git a/include/nlohmann/detail/meta/call_std/end.hpp b/include/nlohmann/detail/meta/call_std/end.hpp index 40c9942b9..dc1895fc4 100644 --- a/include/nlohmann/detail/meta/call_std/end.hpp +++ b/include/nlohmann/detail/meta/call_std/end.hpp @@ -12,6 +12,6 @@ NLOHMANN_JSON_NAMESPACE_BEGIN -NLOHMANN_CAN_CALL_STD_FUNC_IMPL(end); +NLOHMANN_CAN_CALL_STD_FUNC_IMPL(end) NLOHMANN_JSON_NAMESPACE_END diff --git a/include/nlohmann/detail/meta/is_sax.hpp b/include/nlohmann/detail/meta/is_sax.hpp index 8e8a0de24..ac238b0c2 100644 --- a/include/nlohmann/detail/meta/is_sax.hpp +++ b/include/nlohmann/detail/meta/is_sax.hpp @@ -8,7 +8,7 @@ #pragma once -#include // size_t +#include // size_t #include // declval #include // string @@ -70,37 +70,6 @@ using parse_error_function_t = decltype(std::declval().parse_error( std::declval(), std::declval(), std::declval())); -template -struct is_sax -{ - private: - static_assert(is_basic_json::value, - "BasicJsonType must be of type basic_json<...>"); - - using number_integer_t = typename BasicJsonType::number_integer_t; - using number_unsigned_t = typename BasicJsonType::number_unsigned_t; - using number_float_t = typename BasicJsonType::number_float_t; - using string_t = typename BasicJsonType::string_t; - using binary_t = typename BasicJsonType::binary_t; - using exception_t = typename BasicJsonType::exception; - - public: - static constexpr bool value = - is_detected_exact::value && - is_detected_exact::value && - is_detected_exact::value && - is_detected_exact::value && - is_detected_exact::value && - is_detected_exact::value && - is_detected_exact::value && - is_detected_exact::value && - is_detected_exact::value && - is_detected_exact::value && - is_detected_exact::value && - is_detected_exact::value && - is_detected_exact::value; -}; - template struct is_sax_static_asserts { @@ -120,8 +89,6 @@ struct is_sax_static_asserts "Missing/invalid function: bool null()"); static_assert(is_detected_exact::value, "Missing/invalid function: bool boolean(bool)"); - static_assert(is_detected_exact::value, - "Missing/invalid function: bool boolean(bool)"); static_assert( is_detected_exact::value, diff --git a/include/nlohmann/detail/meta/logic.hpp b/include/nlohmann/detail/meta/logic.hpp deleted file mode 100644 index cb50a5d19..000000000 --- a/include/nlohmann/detail/meta/logic.hpp +++ /dev/null @@ -1,54 +0,0 @@ -#pragma once - -#include - -NLOHMANN_JSON_NAMESPACE_BEGIN -namespace detail -{ -#ifdef JSON_HAS_CPP_17 - -template -struct cxpr_or_impl : std::integral_constant < bool, (Booleans || ...) > {}; - -template -struct cxpr_and_impl : std::integral_constant < bool, (Booleans &&...) > {}; - -#else - -template -struct cxpr_or_impl : std::false_type {}; - -template -struct cxpr_or_impl : std::true_type {}; - -template -struct cxpr_or_impl : cxpr_or_impl {}; - -template -struct cxpr_and_impl : std::true_type {}; - -template -struct cxpr_and_impl : cxpr_and_impl {}; - -template -struct cxpr_and_impl : std::false_type {}; - -#endif - -template -struct cxpr_not : std::integral_constant < bool, !Boolean::value > {}; - -template -struct cxpr_or : cxpr_or_impl {}; - -template -struct cxpr_or_c : cxpr_or_impl {}; - -template -struct cxpr_and : cxpr_and_impl {}; - -template -struct cxpr_and_c : cxpr_and_impl {}; - -} // namespace detail -NLOHMANN_JSON_NAMESPACE_END diff --git a/include/nlohmann/detail/meta/type_traits.hpp b/include/nlohmann/detail/meta/type_traits.hpp index 14029c0fc..68bf4ad2c 100644 --- a/include/nlohmann/detail/meta/type_traits.hpp +++ b/include/nlohmann/detail/meta/type_traits.hpp @@ -314,6 +314,13 @@ template struct conjunction : std::conditional(B::value), conjunction, B>::type {}; +// https://en.cppreference.com/w/cpp/types/disjunction +template struct disjunction : std::false_type { }; +template struct disjunction : B { }; +template +struct disjunction +: std::conditional(B::value), B, disjunction>::type {}; + // https://en.cppreference.com/w/cpp/types/negation template struct negation : std::integral_constant < bool, !B::value > { }; @@ -508,9 +515,7 @@ template struct is_range_view_optional_type> : std: template struct is_range_view_optional_type : std::false_type {}; #endif -// std::ranges does not work properly on MinGW due to incomplete C++20 support -// see https://github.com/nlohmann/json/issues/4916 -#if JSON_HAS_RANGES && !defined(__MINGW32__) +#if JSON_HAS_RANGE_VIEW_CONVERSION // SafeToCheck guards against types that trigger circular constraints when // std::ranges::view is evaluated on GCC 12 / libstdc++ 12: @@ -549,7 +554,7 @@ struct is_compatible_array_type_impl < // filter_view) can match BOTH this iterator-based specialization AND the view-based one // below, causing ambiguity. Exclude views here so the two specializations are mutually // exclusive: this one handles plain iterable containers, the other handles views. -#if JSON_HAS_RANGES && !defined(__MINGW32__) +#if JSON_HAS_RANGE_VIEW_CONVERSION && !is_compatible_range_view::value #endif >> @@ -559,7 +564,7 @@ struct is_compatible_array_type_impl < range_value_t>::value; }; -#if JSON_HAS_RANGES && !defined(__MINGW32__) +#if JSON_HAS_RANGE_VIEW_CONVERSION template struct is_compatible_array_type_impl < BasicJsonType, CompatibleArrayType, @@ -635,7 +640,6 @@ struct is_compatible_integer_type_impl < std::is_integral::value&& !std::is_same::value >> { - // is there an assert somewhere on overflows? using RealLimits = std::numeric_limits; using CompatibleLimits = std::numeric_limits; @@ -863,20 +867,7 @@ struct has_capacity : std::integral_constant -struct is_ordered_map -{ - using one = char; - - struct two - { - char x[2]; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) - }; - - template static one test( decltype(&C::capacity) ) ; - template static two test(...); - - enum { value = sizeof(test(nullptr)) == sizeof(char) }; // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg,cppcoreguidelines-use-enum-class) -}; +struct is_ordered_map : has_capacity {}; // to avoid useless casts (see https://github.com/nlohmann/json/issues/2893#issuecomment-889152324) template < typename T, typename U, enable_if_t < !std::is_same::value, int > = 0 > @@ -900,10 +891,8 @@ using all_signed = conjunction...>; template using all_unsigned = conjunction...>; -// there's a disjunction trait in another PR; replace when merged template -using same_sign = std::integral_constant < bool, - all_signed::value || all_unsigned::value >; +using same_sign = disjunction, all_unsigned>; template using never_out_of_range = std::integral_constant < bool, diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index a91bd9361..f8fc9fdb9 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -276,104 +276,6 @@ #include // declval, pair -// #include -// __ _____ _____ _____ -// __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.12.0 -// |_____|_____|_____|_|___| https://github.com/nlohmann/json -// -// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann -// SPDX-License-Identifier: MIT - - - -#include - -// #include -// __ _____ _____ _____ -// __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.12.0 -// |_____|_____|_____|_|___| https://github.com/nlohmann/json -// -// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann -// SPDX-License-Identifier: MIT - - - -// #include - - -NLOHMANN_JSON_NAMESPACE_BEGIN -namespace detail -{ - -template struct make_void -{ - using type = void; -}; -template using void_t = typename make_void::type; - -} // namespace detail -NLOHMANN_JSON_NAMESPACE_END - - -NLOHMANN_JSON_NAMESPACE_BEGIN -namespace detail -{ - -// https://en.cppreference.com/w/cpp/experimental/is_detected -struct nonesuch -{ - nonesuch() = delete; - ~nonesuch() = delete; - nonesuch(nonesuch const&) = delete; - nonesuch(nonesuch const&&) = delete; - void operator=(nonesuch const&) = delete; - void operator=(nonesuch&&) = delete; -}; - -template class Op, - class... Args> -struct detector -{ - using value_t = std::false_type; - using type = Default; -}; - -template class Op, class... Args> -struct detector>, Op, Args...> -{ - using value_t = std::true_type; - using type = Op; -}; - -template class Op, class... Args> -using is_detected = typename detector::value_t; - -template class Op, class... Args> -struct is_detected_lazy : is_detected { }; - -template class Op, class... Args> -using detected_t = typename detector::type; - -template class Op, class... Args> -using detected_or = detector; - -template class Op, class... Args> -using detected_or_t = typename detected_or::type; - -template class Op, class... Args> -using is_detected_exact = std::is_same>; - -template class Op, class... Args> -using is_detected_convertible = - std::is_convertible, To>; - -} // namespace detail -NLOHMANN_JSON_NAMESPACE_END - // #include @@ -2552,10 +2454,12 @@ JSON_HEDLEY_DIAGNOSTIC_POP // libstdc++ < 11 has incomplete C++20 ranges (issue #4440) #elif defined(_GLIBCXX_RELEASE) && _GLIBCXX_RELEASE < 11 #define JSON_HAS_RANGES 0 - // libc++ < 16 has incomplete C++20 ranges (issue #4440) + // clang < 16 with libstdc++ does not implement the ranges customization + // points libstdc++ declares, so its C++20 ranges support is incomplete (issue #5161) #elif defined(__clang__) && !defined(__apple_build_version__) \ && __clang_major__ < 16 && defined(__GLIBCXX__) #define JSON_HAS_RANGES 0 + // libc++ < 16 has incomplete C++20 ranges (issue #4440) #elif defined(_LIBCPP_VERSION) && _LIBCPP_VERSION < 160000 #define JSON_HAS_RANGES 0 // nvcc CUDA 12.0/12.1 chokes on the enable_borrowed_range variable-template @@ -2570,6 +2474,18 @@ JSON_HEDLEY_DIAGNOSTIC_POP #endif #endif +// std::ranges view conversion (to_json/is_compatible_array_type_impl) additionally +// needs to be disabled on MinGW, whose std::ranges support is incomplete +// (issue #4916); this macro combines both conditions so the check and its +// reason are not duplicated at every use site. +#ifndef JSON_HAS_RANGE_VIEW_CONVERSION + #if JSON_HAS_RANGES && !defined(__MINGW32__) + #define JSON_HAS_RANGE_VIEW_CONVERSION 1 + #else + #define JSON_HAS_RANGE_VIEW_CONVERSION 0 + #endif +#endif + #ifndef JSON_HAS_STD_FORMAT #if defined(JSON_HAS_CPP_20) && defined(__cpp_lib_format) #define JSON_HAS_STD_FORMAT 1 @@ -2691,21 +2607,6 @@ JSON_HEDLEY_DIAGNOSTIC_POP -/*! -@brief function to wrap JSON_THROW_MACRO - there can be compilation errors about - there being no arguments to JSON_THROW that depend on template arguments - if this is not used to call JSON_THROW -*/ -template -void templated_json_throw(ExceptionType exception) -{ - JSON_THROW(exception); - - /* JSON_THROW(exception) discards exception and aborts - void cast needed to supress - compilation error if compiled with -Werror and Wunused-parameter */ - (void)exception; -} - /*! @brief macro to briefly define a mapping between an enum and JSON with exception on invalid input @@ -2726,7 +2627,7 @@ void templated_json_throw(ExceptionType exception) return ej_pair.first == e; \ }); \ if (it != std::end(m)) j = it->second; \ - else templated_json_throw(nlohmann::detail::out_of_range::create(410,"enum value out of range for " #ENUM_TYPE, nullptr)); \ + else ::nlohmann::detail::templated_json_throw(nlohmann::detail::out_of_range::create(410,"enum value out of range for " #ENUM_TYPE, nullptr)); \ } \ template \ inline void from_json(const BasicJsonType& j, ENUM_TYPE& e) \ @@ -2741,7 +2642,7 @@ void templated_json_throw(ExceptionType exception) return ej_pair.second == j; \ }); \ if (it != std::end(m)) e = it->first; \ - else templated_json_throw(nlohmann::detail::out_of_range::create(410, nlohmann::detail::concat("enum value out of range for " #ENUM_TYPE ": ", j.dump(-1, ' ', false, nlohmann::detail::error_handler_t::replace)), &j)); \ + else ::nlohmann::detail::templated_json_throw(nlohmann::detail::out_of_range::create(410, nlohmann::detail::concat("enum value out of range for " #ENUM_TYPE ": ", j.dump(-1, ' ', false, nlohmann::detail::error_handler_t::replace)), &j)); \ } // Ugly macros to avoid uglier copy-paste when specializing basic_json. They @@ -3286,30 +3187,6 @@ void templated_json_throw(ExceptionType exception) \ template \ using result_of_##std_name = decltype(std_name(std::declval()...)); \ - } \ - \ - namespace detail2 { \ - struct std_name##_tag \ - { \ - }; \ - \ - template \ - std_name##_tag std_name(T&&...); \ - \ - template \ - using result_of_##std_name = decltype(std_name(std::declval()...)); \ - \ - template \ - struct would_call_std_##std_name \ - { \ - static constexpr auto const value = ::nlohmann::detail:: \ - is_detected_exact::value; \ - }; \ - } /* namespace detail2 */ \ - \ - template \ - struct would_call_std_##std_name : detail2::would_call_std_##std_name \ - { \ } #ifndef JSON_USE_IMPLICIT_CONVERSIONS @@ -3851,6 +3728,31 @@ NLOHMANN_JSON_NAMESPACE_END // #include // #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +// #include + + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +template struct make_void +{ + using type = void; +}; +template using void_t = typename make_void::type; + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END // #include @@ -3922,7 +3824,7 @@ NLOHMANN_JSON_NAMESPACE_END NLOHMANN_JSON_NAMESPACE_BEGIN -NLOHMANN_CAN_CALL_STD_FUNC_IMPL(begin); +NLOHMANN_CAN_CALL_STD_FUNC_IMPL(begin) NLOHMANN_JSON_NAMESPACE_END @@ -3942,13 +3844,84 @@ NLOHMANN_JSON_NAMESPACE_END NLOHMANN_JSON_NAMESPACE_BEGIN -NLOHMANN_CAN_CALL_STD_FUNC_IMPL(end); +NLOHMANN_CAN_CALL_STD_FUNC_IMPL(end) NLOHMANN_JSON_NAMESPACE_END // #include // #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include + +// #include + + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +// https://en.cppreference.com/w/cpp/experimental/is_detected +struct nonesuch +{ + nonesuch() = delete; + ~nonesuch() = delete; + nonesuch(nonesuch const&) = delete; + nonesuch(nonesuch const&&) = delete; + void operator=(nonesuch const&) = delete; + void operator=(nonesuch&&) = delete; +}; + +template class Op, + class... Args> +struct detector +{ + using value_t = std::false_type; + using type = Default; +}; + +template class Op, class... Args> +struct detector>, Op, Args...> +{ + using value_t = std::true_type; + using type = Op; +}; + +template class Op, class... Args> +using is_detected = typename detector::value_t; + +template class Op, class... Args> +struct is_detected_lazy : is_detected { }; + +template class Op, class... Args> +using detected_t = typename detector::type; + +template class Op, class... Args> +using detected_or = detector; + +template class Op, class... Args> +using detected_or_t = typename detected_or::type; + +template class Op, class... Args> +using is_detected_exact = std::is_same>; + +template class Op, class... Args> +using is_detected_convertible = + std::is_convertible, To>; + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END // #include // __ _____ _____ _____ @@ -4314,6 +4287,13 @@ template struct conjunction : std::conditional(B::value), conjunction, B>::type {}; +// https://en.cppreference.com/w/cpp/types/disjunction +template struct disjunction : std::false_type { }; +template struct disjunction : B { }; +template +struct disjunction +: std::conditional(B::value), B, disjunction>::type {}; + // https://en.cppreference.com/w/cpp/types/negation template struct negation : std::integral_constant < bool, !B::value > { }; @@ -4508,9 +4488,7 @@ template struct is_range_view_optional_type> : std: template struct is_range_view_optional_type : std::false_type {}; #endif -// std::ranges does not work properly on MinGW due to incomplete C++20 support -// see https://github.com/nlohmann/json/issues/4916 -#if JSON_HAS_RANGES && !defined(__MINGW32__) +#if JSON_HAS_RANGE_VIEW_CONVERSION // SafeToCheck guards against types that trigger circular constraints when // std::ranges::view is evaluated on GCC 12 / libstdc++ 12: @@ -4549,7 +4527,7 @@ struct is_compatible_array_type_impl < // filter_view) can match BOTH this iterator-based specialization AND the view-based one // below, causing ambiguity. Exclude views here so the two specializations are mutually // exclusive: this one handles plain iterable containers, the other handles views. -#if JSON_HAS_RANGES && !defined(__MINGW32__) +#if JSON_HAS_RANGE_VIEW_CONVERSION && !is_compatible_range_view::value #endif >> @@ -4559,7 +4537,7 @@ struct is_compatible_array_type_impl < range_value_t>::value; }; -#if JSON_HAS_RANGES && !defined(__MINGW32__) +#if JSON_HAS_RANGE_VIEW_CONVERSION template struct is_compatible_array_type_impl < BasicJsonType, CompatibleArrayType, @@ -4635,7 +4613,6 @@ struct is_compatible_integer_type_impl < std::is_integral::value&& !std::is_same::value >> { - // is there an assert somewhere on overflows? using RealLimits = std::numeric_limits; using CompatibleLimits = std::numeric_limits; @@ -4863,20 +4840,7 @@ struct has_capacity : std::integral_constant -struct is_ordered_map -{ - using one = char; - - struct two - { - char x[2]; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) - }; - - template static one test( decltype(&C::capacity) ) ; - template static two test(...); - - enum { value = sizeof(test(nullptr)) == sizeof(char) }; // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg,cppcoreguidelines-use-enum-class) -}; +struct is_ordered_map : has_capacity {}; // to avoid useless casts (see https://github.com/nlohmann/json/issues/2893#issuecomment-889152324) template < typename T, typename U, enable_if_t < !std::is_same::value, int > = 0 > @@ -4900,10 +4864,8 @@ using all_signed = conjunction...>; template using all_unsigned = conjunction...>; -// there's a disjunction trait in another PR; replace when merged template -using same_sign = std::integral_constant < bool, - all_signed::value || all_unsigned::value >; +using same_sign = disjunction, all_unsigned>; template using never_out_of_range = std::integral_constant < bool, @@ -5452,6 +5414,27 @@ class other_error : public exception other_error(int id_, const char* what_arg) : exception(id_, what_arg) {} }; +/*! +@brief helper function to call JSON_THROW from a template +@note JSON_THROW is a macro that, depending on the JSON_THROW_USER / + JSON_TRY_USER / JSON_NOEXCEPTION configuration, may expand to code + that does not reference its argument (e.g. `std::abort()`), which + would trigger a compilation error if the argument's type depends on + a template parameter that is otherwise unused. Wrapping the call in + a templated function avoids this and gives the compiler a single + place to see the (possibly unused) parameter. +*/ +template +void templated_json_throw(ExceptionType exception) +{ + JSON_THROW(exception); + + // JSON_THROW may expand to code that discards its argument (e.g. when + // exceptions are disabled) - the cast below avoids an unused-parameter + // warning with -Werror in that case + (void)exception; +} + } // namespace detail NLOHMANN_JSON_NAMESPACE_END @@ -5521,63 +5504,6 @@ NLOHMANN_JSON_NAMESPACE_END // #include -// #include - - -// #include - - -NLOHMANN_JSON_NAMESPACE_BEGIN -namespace detail -{ -#ifdef JSON_HAS_CPP_17 - -template -struct cxpr_or_impl : std::integral_constant < bool, (Booleans || ...) > {}; - -template -struct cxpr_and_impl : std::integral_constant < bool, (Booleans &&...) > {}; - -#else - -template -struct cxpr_or_impl : std::false_type {}; - -template -struct cxpr_or_impl : std::true_type {}; - -template -struct cxpr_or_impl : cxpr_or_impl {}; - -template -struct cxpr_and_impl : std::true_type {}; - -template -struct cxpr_and_impl : cxpr_and_impl {}; - -template -struct cxpr_and_impl : std::false_type {}; - -#endif - -template -struct cxpr_not : std::integral_constant < bool, !Boolean::value > {}; - -template -struct cxpr_or : cxpr_or_impl {}; - -template -struct cxpr_or_c : cxpr_or_impl {}; - -template -struct cxpr_and : cxpr_and_impl {}; - -template -struct cxpr_and_c : cxpr_and_impl {}; - -} // namespace detail -NLOHMANN_JSON_NAMESPACE_END - // #include // #include @@ -5763,62 +5689,29 @@ inline void from_json(const BasicJsonType& j, std::valarray& l) }); } +// element is not itself a C array: read it directly +template +auto from_json_c_array_element(const BasicJsonType& j, T& e) +-> decltype(e = j.template get(), void()) +{ + e = j.template get(); +} + +// element is itself a C array: recurse one dimension at a time, so any rank is supported template -auto from_json(const BasicJsonType& j, T (&arr)[N]) // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) --> decltype(j.template get(), void()) +void from_json_c_array_element(const BasicJsonType& j, T (&arr)[N]) // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) { for (std::size_t i = 0; i < N; ++i) { - arr[i] = j.at(i).template get(); + from_json_c_array_element(j.at(i), arr[i]); } } -template -auto from_json(const BasicJsonType& j, T (&arr)[N1][N2]) // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) --> decltype(j.template get(), void()) +template +auto from_json(const BasicJsonType& j, T (&arr)[N]) // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) +-> decltype(j.template get::type>(), void()) { - for (std::size_t i1 = 0; i1 < N1; ++i1) - { - for (std::size_t i2 = 0; i2 < N2; ++i2) - { - arr[i1][i2] = j.at(i1).at(i2).template get(); - } - } -} - -template -auto from_json(const BasicJsonType& j, T (&arr)[N1][N2][N3]) // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) --> decltype(j.template get(), void()) -{ - for (std::size_t i1 = 0; i1 < N1; ++i1) - { - for (std::size_t i2 = 0; i2 < N2; ++i2) - { - for (std::size_t i3 = 0; i3 < N3; ++i3) - { - arr[i1][i2][i3] = j.at(i1).at(i2).at(i3).template get(); - } - } - } -} - -template -auto from_json(const BasicJsonType& j, T (&arr)[N1][N2][N3][N4]) // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) --> decltype(j.template get(), void()) -{ - for (std::size_t i1 = 0; i1 < N1; ++i1) - { - for (std::size_t i2 = 0; i2 < N2; ++i2) - { - for (std::size_t i3 = 0; i3 < N3; ++i3) - { - for (std::size_t i4 = 0; i4 < N4; ++i4) - { - arr[i1][i2][i3][i4] = j.at(i1).at(i2).at(i3).at(i4).template get(); - } - } - } - } + from_json_c_array_element(j, arr); } template @@ -5838,20 +5731,33 @@ auto from_json_array_impl(const BasicJsonType& j, std::array& arr, } } +// reserve() is called through this pair (modeled on from_json_object_reserve) +// so from_json_array_impl below has a single body for both ConstructibleArrayType +// that support reserve() and those that don't. +template +auto from_json_array_reserve(ConstructibleArrayType& arr, typename ConstructibleArrayType::size_type size, priority_tag<1> /*unused*/) +-> decltype(arr.reserve(size), void()) +{ + arr.reserve(size); +} + +template +void from_json_array_reserve(ConstructibleArrayType& /*arr*/, std::size_t /*size*/, priority_tag<0> /*unused*/) +{} + template::value, int> = 0> auto from_json_array_impl(const BasicJsonType& j, ConstructibleArrayType& arr, priority_tag<1> /*unused*/) -> decltype( - arr.reserve(std::declval()), j.template get(), void()) { using std::end; ConstructibleArrayType ret; - ret.reserve(j.size()); + from_json_array_reserve(ret, j.size(), priority_tag<1> {}); std::transform(j.begin(), j.end(), std::inserter(ret, end(ret)), [](const BasicJsonType & i) { @@ -5862,27 +5768,6 @@ auto from_json_array_impl(const BasicJsonType& j, ConstructibleArrayType& arr, p arr = std::move(ret); } -template::value, - int> = 0> -inline void from_json_array_impl(const BasicJsonType& j, ConstructibleArrayType& arr, - priority_tag<0> /*unused*/) -{ - using std::end; - - ConstructibleArrayType ret; - std::transform( - j.begin(), j.end(), std::inserter(ret, end(ret)), - [](const BasicJsonType & i) - { - // get() returns *this, this won't call a from_json - // method when value_type is BasicJsonType - return i.template get(); - }); - arr = std::move(ret); -} - template < typename BasicJsonType, typename ConstructibleArrayType, enable_if_t < is_constructible_array_type::value&& @@ -5985,9 +5870,7 @@ inline void from_json(const BasicJsonType& j, ConstructibleObjectType& obj) } // overload for arithmetic types, not chosen for basic_json template arguments -// (BooleanType, etc.); note: Is it really necessary to provide explicit -// overloads for boolean_t etc. in case of a custom BooleanType which is not -// an arithmetic type? +// (BooleanType, etc.) template < typename BasicJsonType, typename ArithmeticType, enable_if_t < std::is_arithmetic::value&& @@ -6083,7 +5966,7 @@ inline void from_json_tuple_impl(const BasicJsonType& j, std::pair& p, p template std::tuple from_json_tuple_impl(const BasicJsonType& j, identity_tag> /*unused*/, priority_tag<2> /*unused*/) { - static_assert(cxpr_and>, is_compatible_reference_type>...>::value, + static_assert(conjunction>, is_compatible_reference_type>...>::value, "Can not return a tuple containing references to types not contained in a Json, try Json::get_to()"); return from_json_tuple_impl_base<1, Args...>(j, index_sequence_for {}); } @@ -6106,10 +5989,10 @@ auto from_json(const BasicJsonType& j, TupleRelated&& t) return from_json_tuple_impl(j, std::forward(t), priority_tag<3> {}); } -template < typename BasicJsonType, typename Key, typename Value, typename Compare, typename Allocator, - typename = enable_if_t < !std::is_constructible < - typename BasicJsonType::string_t, Key >::value >> -inline void from_json(const BasicJsonType& j, std::map& m) +// shared body for std::map/std::unordered_map with a non-string Key: both +// containers are read from an array of [key, value] pairs the same way +template +void from_json_pair_array_to_map(const BasicJsonType& j, MapType& m) { if (JSON_HEDLEY_UNLIKELY(!j.is_array())) { @@ -6122,33 +6005,29 @@ inline void from_json(const BasicJsonType& j, std::map(), p.at(1).template get()); + m.emplace(p.at(0).template get(), p.at(1).template get()); } } +template < typename BasicJsonType, typename Key, typename Value, typename Compare, typename Allocator, + typename = enable_if_t < !std::is_constructible < + typename BasicJsonType::string_t, Key >::value >> +void from_json(const BasicJsonType& j, std::map& m) +{ + from_json_pair_array_to_map(j, m); +} + template < typename BasicJsonType, typename Key, typename Value, typename Hash, typename KeyEqual, typename Allocator, typename = enable_if_t < !std::is_constructible < typename BasicJsonType::string_t, Key >::value >> -inline void from_json(const BasicJsonType& j, std::unordered_map& m) +void from_json(const BasicJsonType& j, std::unordered_map& m) { - if (JSON_HEDLEY_UNLIKELY(!j.is_array())) - { - JSON_THROW(type_error::create(302, concat("type must be array, but is ", j.type_name()), &j)); - } - m.clear(); - for (const auto& p : j) - { - if (JSON_HEDLEY_UNLIKELY(!p.is_array())) - { - JSON_THROW(type_error::create(302, concat("type must be array, but is ", p.type_name()), &p)); - } - m.emplace(p.at(0).template get(), p.at(1).template get()); - } + from_json_pair_array_to_map(j, m); } #if JSON_HAS_FILESYSTEM || JSON_HAS_EXPERIMENTAL_FILESYSTEM -// Workaround for MSVC 19.51 (and possibly later): in large in large cpp files, the compiler may fail to resolve with generic has_from_json (issue #4996) +// Workaround for MSVC 19.51 (and possibly later): in large cpp files, the compiler may fail to resolve with generic has_from_json (issue #4996) template struct has_from_json : std::true_type {}; @@ -6844,7 +6723,7 @@ struct external_constructor template < typename BasicJsonType, typename CompatibleArrayType, enable_if_t < !std::is_same::value -#if JSON_HAS_RANGES && !defined(__MINGW32__) +#if JSON_HAS_RANGE_VIEW_CONVERSION && !is_compatible_range_view::value #endif , int > = 0 > @@ -6888,9 +6767,7 @@ struct external_constructor j.assert_invariant(); } - // std::ranges does not work properly on MinGW due to incomplete C++20 support - // see https://github.com/nlohmann/json/issues/4916 -#if JSON_HAS_RANGES && !defined(__MINGW32__) +#if JSON_HAS_RANGE_VIEW_CONVERSION template>::value, int> = 0> static void construct(BasicJsonType& j, CompatibleArrayType && arr) @@ -7050,7 +6927,7 @@ template < typename BasicJsonType, typename CompatibleArrayType, !std::is_same::value&& !is_compatible_binary_type::value&& !is_basic_json::value -#if JSON_HAS_RANGES && !defined(__MINGW32__) +#if JSON_HAS_RANGE_VIEW_CONVERSION && !is_compatible_range_view::value #endif , @@ -7060,7 +6937,7 @@ inline void to_json(BasicJsonType& j, const CompatibleArrayType& arr) external_constructor::construct(j, arr); } -#if JSON_HAS_RANGES && !defined(__MINGW32__) +#if JSON_HAS_RANGE_VIEW_CONVERSION template < typename BasicJsonType, typename T, enable_if_t < is_compatible_range_view>::value && !is_compatible_string_type>::value @@ -13449,7 +13326,7 @@ NLOHMANN_JSON_NAMESPACE_END -#include // size_t +#include // size_t #include // declval #include // string @@ -13514,37 +13391,6 @@ using parse_error_function_t = decltype(std::declval().parse_error( std::declval(), std::declval(), std::declval())); -template -struct is_sax -{ - private: - static_assert(is_basic_json::value, - "BasicJsonType must be of type basic_json<...>"); - - using number_integer_t = typename BasicJsonType::number_integer_t; - using number_unsigned_t = typename BasicJsonType::number_unsigned_t; - using number_float_t = typename BasicJsonType::number_float_t; - using string_t = typename BasicJsonType::string_t; - using binary_t = typename BasicJsonType::binary_t; - using exception_t = typename BasicJsonType::exception; - - public: - static constexpr bool value = - is_detected_exact::value && - is_detected_exact::value && - is_detected_exact::value && - is_detected_exact::value && - is_detected_exact::value && - is_detected_exact::value && - is_detected_exact::value && - is_detected_exact::value && - is_detected_exact::value && - is_detected_exact::value && - is_detected_exact::value && - is_detected_exact::value && - is_detected_exact::value; -}; - template struct is_sax_static_asserts { @@ -13564,8 +13410,6 @@ struct is_sax_static_asserts "Missing/invalid function: bool null()"); static_assert(is_detected_exact::value, "Missing/invalid function: bool boolean(bool)"); - static_assert(is_detected_exact::value, - "Missing/invalid function: bool boolean(bool)"); static_assert( is_detected_exact::value, @@ -18725,9 +18569,11 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci static_assert(is_basic_json::type>::value, "iter_impl only accepts (const) basic_json"); // superficial check for the LegacyBidirectionalIterator named requirement - static_assert(std::is_base_of::value - && std::is_base_of::iterator_category>::value, - "basic_json iterator assumes array and object type iterators satisfy the LegacyBidirectionalIterator named requirement."); + // note: only array_t::iterator is checked here; object_t::iterator may be + // a forward-only iterator as long as reverse iteration and operator-- + // are never used on it + static_assert(std::is_base_of::iterator_category>::value, + "basic_json iterator assumes array type iterators satisfy the LegacyBidirectionalIterator named requirement."); public: /// The std::iterator class template (used as a base class to provide typedefs) is deprecated in C++17. @@ -19869,6 +19715,72 @@ class json_pointer } private: + /*! + @brief result of @ref parse_array_index + + @ref array_index maps each value to the corresponding parse_error/out_of_range + exception; @ref contains and @ref get_checked_or_null, which must not throw for + an out-of-range or unrepresentable index, switch on it directly instead. + */ + enum class array_index_status + { + ok, ///< @a s is a valid, representable array index + leading_zero, ///< @a s begins with '0' but has more than one character + not_a_number, ///< @a s does not begin with a digit + unresolved, ///< @a s could not be converted to an integer + exceeds_size_type ///< @a s converts to an integer that exceeds size_type + }; + + /*! + @param[in] s reference token to be converted into an array index + @param[out] idx the integer representation of @a s if @ref array_index_status::ok + is returned; left unchanged otherwise + + @return whether @a s is a valid array index, and if not, why + + @note this function never throws; @ref array_index and the callers that must not + throw (@ref contains, @ref get_checked_or_null) build on it instead of each + re-implementing the RFC 6901 digit rules and the @a size_type range check + */ + template + static array_index_status parse_array_index(const string_t& s, typename BasicJsonType::size_type& idx) noexcept + { + using size_type = typename BasicJsonType::size_type; + + // error condition (cf. RFC 6901, Sect. 4) + if (JSON_HEDLEY_UNLIKELY(s.size() > 1 && s[0] == '0')) + { + return array_index_status::leading_zero; + } + + // error condition (cf. RFC 6901, Sect. 4) + if (JSON_HEDLEY_UNLIKELY(s.size() > 1 && !(s[0] >= '1' && s[0] <= '9'))) + { + return array_index_status::not_a_number; + } + + const char* p = s.data(); + char* p_end = nullptr; // NOLINT(misc-const-correctness) + errno = 0; // strtoull doesn't reset errno + const unsigned long long res = std::strtoull(p, &p_end, 10); // NOLINT(runtime/int) + if (p == p_end // invalid input or empty string + || errno == ERANGE // out of range + || JSON_HEDLEY_UNLIKELY(static_cast(p_end - p) != s.size())) // incomplete read + { + return array_index_status::unresolved; + } + + // the index does not fit into size_type; on 64-bit platforms this is + // only SIZE_MAX itself (see #2203 and #5395) + if (res >= static_cast((std::numeric_limits::max)())) // NOLINT(runtime/int) + { + return array_index_status::exceeds_size_type; + } + + idx = static_cast(res); + return array_index_status::ok; + } + /*! @param[in] s reference token to be converted into an array index @@ -19882,39 +19794,23 @@ class json_pointer template static typename BasicJsonType::size_type array_index(const string_t& s) { - using size_type = typename BasicJsonType::size_type; - - // error condition (cf. RFC 6901, Sect. 4) - if (JSON_HEDLEY_UNLIKELY(s.size() > 1 && s[0] == '0')) + typename BasicJsonType::size_type idx{}; + switch (parse_array_index(s, idx)) { - JSON_THROW(detail::parse_error::create(106, 0, detail::concat("array index '", s, "' must not begin with '0'"), nullptr)); + case array_index_status::leading_zero: + JSON_THROW(detail::parse_error::create(106, 0, detail::concat("array index '", s, "' must not begin with '0'"), nullptr)); + case array_index_status::not_a_number: + JSON_THROW(detail::parse_error::create(109, 0, detail::concat("array index '", s, "' is not a number"), nullptr)); + case array_index_status::unresolved: + JSON_THROW(detail::out_of_range::create(404, detail::concat("unresolved reference token '", s, "'"), nullptr)); + case array_index_status::exceeds_size_type: + JSON_THROW(detail::out_of_range::create(410, detail::concat("array index ", s, " exceeds size_type"), nullptr)); + case array_index_status::ok: + default: + break; } - // error condition (cf. RFC 6901, Sect. 4) - if (JSON_HEDLEY_UNLIKELY(s.size() > 1 && !(s[0] >= '1' && s[0] <= '9'))) - { - JSON_THROW(detail::parse_error::create(109, 0, detail::concat("array index '", s, "' is not a number"), nullptr)); - } - - const char* p = s.data(); - char* p_end = nullptr; // NOLINT(misc-const-correctness) - errno = 0; // strtoull doesn't reset errno - const unsigned long long res = std::strtoull(p, &p_end, 10); // NOLINT(runtime/int) - if (p == p_end // invalid input or empty string - || errno == ERANGE // out of range - || JSON_HEDLEY_UNLIKELY(static_cast(p_end - p) != s.size())) // incomplete read - { - JSON_THROW(detail::out_of_range::create(404, detail::concat("unresolved reference token '", s, "'"), nullptr)); - } - - // the index does not fit into size_type; on 64-bit platforms this is - // only SIZE_MAX itself (see #2203 and #5395) - if (res >= static_cast((std::numeric_limits::max)())) // NOLINT(runtime/int) - { - JSON_THROW(detail::out_of_range::create(410, detail::concat("array index ", s, " exceeds size_type"), nullptr)); - } - - return static_cast(res); + return idx; } JSON_PRIVATE_UNLESS_TESTED: @@ -20219,63 +20115,6 @@ class json_pointer return *ptr; } - /*! - @throw parse_error.106 if an array index begins with '0' - @throw parse_error.109 if an array index was not a number - @throw out_of_range.402 if the array index '-' is used - @throw out_of_range.404 if the JSON pointer can not be resolved - */ - template - const BasicJsonType& get_checked(const BasicJsonType* ptr) const - { - for (const auto& reference_token : reference_tokens) - { - switch (ptr->type()) - { - case detail::value_t::object: - { - // note: at performs range check - ptr = &ptr->at(reference_token); - break; - } - - case detail::value_t::array: - { - if (JSON_HEDLEY_UNLIKELY(reference_token == "-")) - { - // "-" always fails the range check - JSON_THROW(detail::out_of_range::create(402, detail::concat( - "array index '-' (", std::to_string(ptr->m_data.m_value.array->size()), - ") is out of range"), ptr)); - } - - const auto idx = array_index(reference_token); - // Bounds check before access to avoid exception with JSON_NOEXCEPTION - if (JSON_HEDLEY_UNLIKELY(idx >= ptr->m_data.m_value.array->size())) - { - JSON_THROW(detail::out_of_range::create(401, detail::concat( - "array index ", std::to_string(idx), " is out of range"), ptr)); - } - ptr = &ptr->operator[](idx); - break; - } - - case detail::value_t::null: - case detail::value_t::string: - case detail::value_t::boolean: - case detail::value_t::number_integer: - case detail::value_t::number_unsigned: - case detail::value_t::number_float: - case detail::value_t::binary: - case detail::value_t::discarded: - default: - JSON_THROW(detail::out_of_range::create(404, detail::concat("unresolved reference token '", reference_token, "'"), ptr)); - } - } - - return *ptr; - } - /*! @brief return a pointer to the pointed to value, or `nullptr` if the pointer cannot be resolved because a key is missing, an array @@ -20314,29 +20153,24 @@ class json_pointer return nullptr; } - // tokens that array_index() rejects with parse_error.106/109 - // are passed on to it; all other tokens that it would reject - // with out_of_range.404/410 are detected here, so that this - // also works without exceptions - if (JSON_HEDLEY_UNLIKELY(reference_token.size() > 1 && !(reference_token[0] >= '1' && reference_token[0] <= '9'))) + // a malformed index throws parse_error.106/109; an + // index that is syntactically valid but cannot be + // represented (out_of_range.404/410) is treated like an + // out-of-range index below + typename BasicJsonType::size_type idx{}; + switch (parse_array_index(reference_token, idx)) { - static_cast(array_index(reference_token)); // throws parse_error.106/109 + case array_index_status::leading_zero: + JSON_THROW(detail::parse_error::create(106, 0, detail::concat("array index '", reference_token, "' must not begin with '0'"), nullptr)); + case array_index_status::not_a_number: + JSON_THROW(detail::parse_error::create(109, 0, detail::concat("array index '", reference_token, "' is not a number"), nullptr)); + case array_index_status::unresolved: + case array_index_status::exceeds_size_type: + return nullptr; + case array_index_status::ok: + default: + break; } - if (JSON_HEDLEY_UNLIKELY(reference_token.empty() || !std::all_of(reference_token.begin(), reference_token.end(), [](const char c) - { - return c >= '0' && c <= '9'; - }))) - { - return nullptr; - } - errno = 0; // strtoull() does not reset errno on success - char* p_end = nullptr; // NOLINT(misc-const-correctness) - const unsigned long long magnitude = std::strtoull(reference_token.data(), &p_end, 10); // NOLINT(runtime/int) - if (JSON_HEDLEY_UNLIKELY(errno == ERANGE || magnitude >= static_cast((std::numeric_limits::max)()))) // NOLINT(runtime/int) - { - return nullptr; - } - const auto idx = static_cast(magnitude); if (JSON_HEDLEY_UNLIKELY(idx >= ptr->m_data.m_value.array->size())) { @@ -20363,8 +20197,8 @@ class json_pointer } /*! - @throw parse_error.106 if an array index begins with '0' - @throw parse_error.109 if an array index was not a number + @note unlike array_index(), this never throws: a malformed or unrepresentable + array index reference token is treated like a missing key (see #5395) */ template bool contains(const BasicJsonType* ptr) const @@ -20392,49 +20226,17 @@ class json_pointer // "-" always fails the range check return false; } - if (JSON_HEDLEY_UNLIKELY(reference_token.empty())) - { - // an empty reference token is not an array index; array_index() - // would throw out_of_range.404 -- contains() must not throw (see #5395) - return false; - } - if (JSON_HEDLEY_UNLIKELY(reference_token.size() == 1 && !('0' <= reference_token[0] && reference_token[0] <= '9'))) - { - // invalid char - return false; - } - if (JSON_HEDLEY_UNLIKELY(reference_token.size() > 1)) - { - if (JSON_HEDLEY_UNLIKELY(!('1' <= reference_token[0] && reference_token[0] <= '9'))) - { - // the first char should be between '1' and '9' - return false; - } - for (std::size_t i = 1; i < reference_token.size(); i++) - { - if (JSON_HEDLEY_UNLIKELY(!('0' <= reference_token[i] && reference_token[i] <= '9'))) - { - // other char should be between '0' and '9' - return false; - } - } - } - // the reference token consists only of digits at this point (cf. checks - // above); however, its numeric value might not be representable, in which - // case array_index() would throw out_of_range.404/410 -- contains() must - // not throw (see #5395), so such a reference token is treated as "not found" - errno = 0; // strtoull() does not reset errno on success - char* p_end = nullptr; // NOLINT(misc-const-correctness) - const unsigned long long magnitude = std::strtoull(reference_token.data(), &p_end, 10); // NOLINT(runtime/int) - if (JSON_HEDLEY_UNLIKELY(errno == ERANGE // the value exceeds ULLONG_MAX - || magnitude >= static_cast((std::numeric_limits::max)()))) // NOLINT(runtime/int) + // any parse failure (malformed index, or one that is syntactically + // valid but not representable as size_type) means the reference + // token cannot denote an existing array element -- contains() must + // not throw (see #5395), so it is treated as "not found" + typename BasicJsonType::size_type idx{}; + if (JSON_HEDLEY_UNLIKELY(parse_array_index(reference_token, idx) != array_index_status::ok)) { - // the array index cannot be represented as size_type return false; } - const auto idx = array_index(reference_token); if (idx >= ptr->size()) { // index out of range @@ -34096,6 +33898,7 @@ struct formatter // NOLINT(cert-dcl58-c #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM #undef JSON_HAS_THREE_WAY_COMPARISON #undef JSON_HAS_RANGES + #undef JSON_HAS_RANGE_VIEW_CONVERSION #undef JSON_HAS_STD_FORMAT #undef JSON_HAS_STATIC_RTTI #undef JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON diff --git a/tests/src/unit-conversions.cpp b/tests/src/unit-conversions.cpp index df8685529..5f83a9253 100644 --- a/tests/src/unit-conversions.cpp +++ b/tests/src/unit-conversions.cpp @@ -430,6 +430,37 @@ TEST_CASE("value conversion") CHECK(std::equal(std::begin(nbs[0][0][0]), std::end(nbs[1][1][1]), std::begin(nbs2[0][0][0]))); } + SECTION("built-in arrays: 5D") + { + // NOLINTBEGIN(misc-const-correctness,cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + const int nbs[1][1][1][2][2] = {{{{{0, 1}, {2, 3}}}}}; + int nbs2[1][1][1][2][2] = {{{{{0, 0}, {0, 0}}}}}; + // NOLINTEND(misc-const-correctness,cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + + const json j2 = nbs; + j2.get_to(nbs2); + CHECK(std::equal(std::begin(nbs[0][0][0][0]), std::end(nbs[0][0][0][1]), std::begin(nbs2[0][0][0][0]))); + } + + SECTION("built-in arrays: mismatched shape") + { + // NOLINTBEGIN(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + int nbs2[2][3] = {{0, 0, 0}, {0, 0, 0}}; + // NOLINTEND(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + + SECTION("not an array") + { + const json j2 = 42; + CHECK_THROWS_WITH_AS(j2.get_to(nbs2), "[json.exception.type_error.304] cannot use at() with number", json::type_error&); + } + + SECTION("too few elements") + { + const json j2 = {{0, 1, 2}}; + CHECK_THROWS_WITH_AS(j2.get_to(nbs2), "[json.exception.out_of_range.401] array index 1 is out of range", json::out_of_range&); + } + } + SECTION("std::deque") { std::deque a{"previous", "value"}; @@ -1748,6 +1779,34 @@ NLOHMANN_JSON_SERIALIZE_ENUM_STRICT(StrictTaskState, {STRICT_TS_COMPLETED, "completed"}, }) +// regression test for #5708 item 2: NLOHMANN_JSON_SERIALIZE_ENUM_STRICT must not rely on +// unqualified lookup of a helper name that a user's own namespace may also declare +namespace ns_with_colliding_name +{ +// NOLINTNEXTLINE(misc-use-internal-linkage) - used to shadow the library's internal helper name +inline void templated_json_throw(int /*unused*/) {} + +enum class colliding_enum { a, b }; + +// NOLINTNEXTLINE(misc-use-internal-linkage,misc-const-correctness,cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) - false positive +NLOHMANN_JSON_SERIALIZE_ENUM_STRICT(colliding_enum, +{ + {colliding_enum::a, "a"}, + {colliding_enum::b, "b"} +}) +} // namespace ns_with_colliding_name + +TEST_CASE("NLOHMANN_JSON_SERIALIZE_ENUM_STRICT in a namespace with a colliding name") +{ + using ns_with_colliding_name::colliding_enum; + + CHECK(json(colliding_enum::a) == "a"); + CHECK(colliding_enum::b == json("b")); + + json _; + CHECK_THROWS_WITH_AS(_ = json("nope").get(), "[json.exception.out_of_range.410] enum value out of range for colliding_enum: \"nope\"", json::out_of_range&); +} + TEST_CASE("Strict JSON to enum mapping") { SECTION("enum class") diff --git a/tests/src/unit-type_traits.cpp b/tests/src/unit-type_traits.cpp index 6dc166d03..a3937bf2e 100644 --- a/tests/src/unit-type_traits.cpp +++ b/tests/src/unit-type_traits.cpp @@ -10,10 +10,14 @@ #if JSON_TEST_USING_MULTIPLE_HEADERS #include + #include #else #include #endif +#include +#include + TEST_CASE("type traits") { SECTION("is_c_string") @@ -83,4 +87,12 @@ TEST_CASE("type traits") } } } + + SECTION("is_ordered_map") + { + using nlohmann::detail::is_ordered_map; + + CHECK(is_ordered_map>::value); + CHECK_FALSE(is_ordered_map>::value); + } } From a0b71e272006c400703daeb2f4dc3248e47ddb01 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 4 Oct 2026 11:57:41 +0200 Subject: [PATCH 15/22] Deduplicate basic_json internals; make insert(pos, json&&) move (#5727) * Remove unused private aliases from basic_json The private aliases primitive_iterator_t, internal_iterator and output_adapter_t are not used anywhere: iter_impl, binary_writer and the tests refer to the detail:: names directly. As the aliases are private, no user or derived class can depend on them. The internal_iterator.hpp include stays because iter_impl needs it. Part of #5724 Signed-off-by: Niels Lohmann * Fix meta()'s dead, syntactically invalid HP aCC branch The HP aCC branch of basic_json::meta() was missing a semicolon and has therefore never compiled; adding only a semicolon would also make it throw type_error.305, since it assigned a plain string to result["compiler"] and then indexed into it like the other branches do into an object. Make the branch consistent with the others by assigning an object with "family" and "version" keys, narrow the condition to __HP_aCC (a C compiler cannot build this header-only library), and fix meta.md, which documented the old (impossible) plain-string behavior. Part of #5724 Signed-off-by: Niels Lohmann * Stop the noexcept null constructor delegating to a throwing one basic_json(std::nullptr_t) delegated to basic_json(value_t), whose underlying json_value(value_t) constructor allocates for other types and can therefore throw, which is why the noexcept had a NOLINT(bugprone-exception-escape). The delegated-to constructor also called assert_invariant() a second time. The default member initializers of data already produce the same null state (a value-initialized, i.e. zeroed, union with object == nullptr), so the delegation and its NOLINT can simply be dropped. Part of #5724 Signed-off-by: Niels Lohmann * Remove four cppcheck accessForwarded suppressions in the move constructor basic_json(basic_json&&) built its base subobject with std::forward(other), so cppcheck saw the whole of other as forwarded and flagged every subsequent access to it as accessForwarded, three of them still marked "TODO check". Only the base subobject is actually moved from; cast explicitly to the base type instead, the way ordered_map already does, so cppcheck can tell the two are unrelated. Behavior is unchanged: for a non-reference T, std::forward(x) is defined as static_cast(x). Part of #5724 Signed-off-by: Niels Lohmann * Remove stale cppcheck suppressions and name the local parser in parse() Running the pinned cppcheck (ci_cppcheck's invocation) without --inline-suppr across all configurations reports no syntaxError, no ignoredReturnValue and no assertWithSideEffect, so the corresponding suppressions in json_fwd.hpp, string_concat.hpp and assert_invariant() no longer match anything (json_fwd.hpp's is kept, since downstream users who run an older cppcheck against it could still hit the warning it once silenced). The three basic_json::parse() overloads still trigger a false-positive accessMoved/accessForwarded because they build a temporary parser and call .parse() on it in the same expression; giving that parser a name makes the warning go away without changing behavior, and removes the last of the inline suppressions on these functions. Part of #5724 Signed-off-by: Niels Lohmann * Deduplicate the string/binary cleanup in the two erase() overloads erase(pos) and erase(first, last) each carried a byte-identical 14-line block that destroys and deallocates a string or binary value before resetting the type to null. That reimplements the string/binary cases of json_value::destroy(), so any future change to how those values are freed would have to be made in three places instead of one. Both overloads now just call destroy() and reset the union; for the other primitive types (boolean, numbers) destroy() is a no-op, so behavior is unchanged. Also fix erase(first, last)'s error-path branch hint, which used JSON_HEDLEY_LIKELY where erase(pos), the iterator-range constructor, and every other error path in the class use JSON_HEDLEY_UNLIKELY. This only affects code layout, not semantics. Part of #5724 Signed-off-by: Niels Lohmann * Fix stale and copy-pasted comments in basic_json Several comments no longer match the code: the class invariant and assert_invariant()'s doc still named the members m_value/m_type (now m_data.m_value/m_data.m_type) and did not mention the binary invariant that assert_invariant() already checks; the json_value note and the get() @tparam list omitted binary_t even though binary is a variable-length, pointer-stored type like the others; the key-based value() overload's brief said "via JSON Pointer", which is the other overload; and swap(binary_t&)/swap(binary_t::container_type&) both carried "swap only works for strings", copied from swap(string_t&). Comment-only change; behavior, the public API and the ABI are unchanged. The private get_impl() doxygen and the emplace() comments that border #5585's hunk are intentionally left alone. Part of #5724 Signed-off-by: Niels Lohmann * Deduplicate the 16 copied from_cbor/msgpack/ubjson/bjdata/bon8/bson bodies Each of the 16 binary deserialization overloads (from_cbor, from_msgpack, from_ubjson, from_bjdata, from_bon8, from_bson, each in an InputType&& and an iterator/sentinel version, plus the deprecated span overloads of from_cbor/from_msgpack/from_ubjson/from_bson) had the same body, differing only in the input_format_t value. Every copy built a temporary binary_reader from std::move(ia) and called sax_parse on it in the same expression, which also produced a false-positive cppcheck accessMoved on all 16 lines and needed a NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) on the four span overloads. Add a private from_binary_impl() helper that builds the reader as a named local instead, and make each of the 16 overloads a one-line forward to it. All public signatures, default arguments, JSON_HEDLEY_WARN_UNUSED_RESULT and JSON_HEDLEY_DEPRECATED_FOR attributes are unchanged, tag_handler keeps defaulting to cbor_tag_handler_t::error for the non-CBOR formats (matching binary_reader::sax_parse's own default), and the helper is placed in the existing private section before the binary section banner rather than between the from_* overloads, so from_binary_impl() itself does not collide with #5688's insertion point. Collapsing the from_bjdata/from_bon8 bodies into one-line forwards does rewrite the "return result; }" context lines that #5688 inserts its two deprecated overloads after, so that PR will need a small manual rebase (reinserting its overloads after the new one-line bodies) rather than applying cleanly. Part of #5724 Signed-off-by: Niels Lohmann * Make insert(pos, basic_json&&) move its argument instead of copying it insert(const_iterator pos, basic_json&& val) delegated to insert(pos, val), but val is a named rvalue reference, so inside the function it is an lvalue: the call always resolved to insert(const_iterator, const basic_json&) and deep-copied the value. This has been the case since the overload was introduced, in every release. push_back(basic_json&&), by contrast, already moves. Give the rvalue overload its own body with the same two checks (type_error.309, invalid_iterator.202), then move the argument into a local before inserting it. Moving into a local first, rather than inserting std::move(val) directly, keeps this safe even when val aliases an element of the same array (e.g. arr.insert(arr.begin(), std::move(arr[1]))), since std::vector::insert(pos, T&&) is not guaranteed to handle an argument that aliases one of its own elements. This is a deliberate, small behavior change: the moved-from argument now ends up null afterwards, the same as after push_back(&&), instead of keeping its old value unchanged. No signature changes, so the public API and ABI are unaffected. Add unit-modifiers coverage for the moved-from state and for self-aliasing insertion, both with and without reallocation of the underlying array. Part of #5724 Signed-off-by: Niels Lohmann * Deduplicate object key lookup and checked at() access at()/find()/count()/contains()/const operator[]/erase_internal() each repeated the raw object lookup (m_value.object->find(key)), and the six at() overloads additionally repeated the type_error.304 check and out_of_range.401/403 throw. Route them all through two new private helpers, object_lookup()/object_at() (plus array_at() for the index overloads of at()), templated on the constness of the receiver so one body serves both the const and non-const overload. count() is left untouched, since it already goes through object_t::count() rather than a second find(). The at(KeyType&&) overloads used to forward the same key twice: once into object->find() and again, on the not-found path, into the string_t() conversion for the exception message. object_at() now forwards it only into the lookup and reuses the (unmoved) key for the message. clang-tidy 22 (Docker silkeh/clang:22) still flags that reuse under bugprone-use-after-move/hicpp-invalid-access-moved even with the single forward, since it cannot see that object_t::find() (a plain std::map or ordered_map) never actually moves from its argument; add a NOLINTNEXTLINE with that reasoning rather than avoid the pattern. No signature, exception id/message, or set_parent() behavior changes. Overlaps #5689, #5705, #5606, #5687 and #5585, which touch the same hunks; whichever of this commit and those PRs lands second will need a small rebase. Signed-off-by: Niels Lohmann * Deduplicate the lookup/default/throw body of value() The six non-deprecated value() overloads each held a full copy of the same body: the four key-based overloads looked up the key and either returned the found element converted to the requested type or the default value (throwing type_error.306 if this is not an object), and the two json_pointer overloads did the same via ptr.get_checked_or_null(), throwing type_error.306 unless is_structured(). Replace the duplicated bodies with two private helpers, value_member() and value_pointee(), that return a const basic_json* (null when not found) and do the type check/throw once each. Every value() overload now just picks between the found pointer's get() and the default. Same signatures, template parameters, SFINAE conditions, exception id, message and this context on every overload. Overlaps #5689 (routes find() through lookup_key()) and #5705 (adds a deleted integral-key value() next to these overloads); whichever of this commit and those PRs lands second will need a small rebase. #5724 item 4 Signed-off-by: Niels Lohmann * Deduplicate the null-to-container conversion into convert_null_to() Nine sites wrote out the same "turn a null value into an empty array or object" logic with two different idioms: operator[](size_type), operator[](key_type), operator[](KeyType&&) and update() set m_type then assigned m_value.array/object directly via create(), while the three push_back() overloads, emplace_back() and emplace() set m_type then assigned m_value = value_t::array/object (going through json_value's converting constructor and a temporary). Both idioms end up calling create() and produce the same state, just via a different path; both also share a latent exception-safety bug, since m_type is written before the (possibly throwing) allocation, so a throwing allocator leaves m_type == array/object with a null pointer behind it, violating the class invariant and crashing on the next access to, or destruction of, the value. Add a private convert_null_to(value_t) helper and call it from all nine sites. Unlike the idioms it replaces, it allocates the container first and only then writes m_type, so a throwing allocation leaves the value as a valid null instead of a mistyped, half-constructed one; verified with a throwing allocator (see unit-allocator.cpp's bad_allocator) that j["x"] = ... on a null j now stays null, and no longer trips assert_invariant()/crashes, when create() throws. Same allocator usage and assert_invariant() call as before, otherwise. Overlaps #5585, which reorders these same nine blocks for exception safety; whichever of this commit and that PR lands second will need a small rebase. Signed-off-by: Niels Lohmann * Deduplicate the linear key search in ordered_map emplace, at, erase(key), count and find each repeated the same "for (auto it = begin(); it != end(); ++it) if (m_compare(it->first, key)) ..." loop (15 copies across their key_type and transparent KeyType&& overloads), and both erase(key) overloads additionally repeated the exception-sensitive in-place reconstruction (destroy, placement-new, pop_back) used to remove an element while keeping the const Key non-movable. Add two private helpers: find_impl(Self&, KeyType&&), a static member template that runs the search once for either constness of the receiver, and erase_at(iterator), which keeps the existing pop_back-based reconstruction instead of switching to erase()/resize() (which would add a DefaultInsertable requirement). Route find, at, count, emplace, insert(const value_type&) and both erase(key) overloads through them. Same signatures, is_usable_as_key_type constraints and exception messages/types. Overlaps #5609 and #5685, which both rewrite emplace (#5609 also touches insert and adds private members at the end of the class); whichever of this commit and those PRs lands second will need a rebase. #5724 item 6 Signed-off-by: Niels Lohmann * Re-enable bugprone-use-after-move/hicpp-invalid-access-moved These two checks (and portability-template-virtual-member-function) were disabled in #4489 (November 2024) "only removed to get the CI going". portability-template-virtual-member-function is a separate, still-open cleanup (#5725 item 3 on its own branch) and stays disabled here; this commit only re-enables the move/forward checks and cleans up what they flag on this branch. The move constructor (json.hpp) already casts to the base type instead of forwarding the whole object (#5724 item 9), so it no longer trips either check. The at(KeyType&&) double-forward this check used to flag was reduced to a single forward with the now-unforwarded reuse annotated by a NOLINTNEXTLINE in #5724 item 3's object_at() helper (clang-tidy 22 still flags that reuse even after a single forward; see that commit's message). What is left here: - from_json_inplace_array_impl(), from_json_tuple_impl_base() and the std::pair overload of from_json_tuple_impl() forwarded j into every j.at(...) call in a pack expansion or a pair of calls. at() has no ref-qualified overloads, so the forward was a no-op; call j.at(...) directly. - container_input_adapter_factory::create() forwards container twice on purpose, into begin() and end(), so both see the same value category and produce matching iterator types. Annotate it with NOLINTNEXTLINE and a comment instead of changing it. - unit-class_parser.cpp's "move constructor resets the moved-from value to npos" test still pointed at the pre-static_cast move constructor by line number and mentioned the cppcheck-suppress annotation that #5724 item 9 already removed; update the comment. No behavior change anywhere in include/. Verified with clang-tidy 22.1.8 (Docker silkeh/clang:22, --platform linux/amd64) against a TU including json.hpp with the repo's .clang-tidy: bugprone-use-after-move and hicpp-invalid-access-moved report nothing unsuppressed. Overlaps #5737 (open PR for the rest of #5725 item 3: the from_json.hpp/input_adapters.hpp cleanup above, and portability-template-virtual-member-function), which currently keeps both checks disabled pending this move-constructor change; whichever of this commit and that PR lands second will need a small rebase of .clang-tidy. Signed-off-by: Niels Lohmann * Make convert_null_to() take the container type as a template argument Passing array_t or object_t instead of a value_t makes an invalid target a compile error instead of a runtime assertion, and removes the branch. Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann --- .clang-tidy | 9 - docs/mkdocs/docs/api/basic_json/meta.md | 2 +- .../nlohmann/detail/input/input_adapters.hpp | 2 +- include/nlohmann/detail/string_concat.hpp | 1 - include/nlohmann/json.hpp | 644 +++++++----------- include/nlohmann/ordered_map.hpp | 183 ++--- tests/src/unit-class_parser.cpp | 8 +- tests/src/unit-modifiers.cpp | 43 ++ 8 files changed, 366 insertions(+), 526 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index fa3e03ae3..a16136785 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -1,18 +1,9 @@ -# bugprone-use-after-move (hicpp-invalid-access-moved is its alias) still flags -# the basic_json move constructor, which forwards the whole object to its base -# class (#5724), and two forwards in the error-message construction of -# at(KeyType&&) (json.hpp, both overloads: find(std::forward(key)) -# followed by string_t(std::forward(key)) in the throw), which #5689 -# rewrites. Re-enable both checks once those changes have landed. # portability-avoid-pragma-once: kept disabled on purpose. #pragma once is accepted # by every supported compiler, and tools/amalgamate/amalgamate.py strips it from # single_include, so there is nothing left to fix here. Checks: '*, - -bugprone-use-after-move, - -hicpp-invalid-access-moved, - -altera-id-dependent-backward-branch, -altera-struct-pack-align, -altera-unroll-loops, diff --git a/docs/mkdocs/docs/api/basic_json/meta.md b/docs/mkdocs/docs/api/basic_json/meta.md index 55476c5f3..9b1ac2a4e 100644 --- a/docs/mkdocs/docs/api/basic_json/meta.md +++ b/docs/mkdocs/docs/api/basic_json/meta.md @@ -13,7 +13,7 @@ JSON object holding version information | key | description | |-------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `compiler` | Information on the used compiler. It is an object with the following keys: `c++` (the used C++ standard), `family` (the compiler family; possible values are `clang`, `icc`, `gcc`, `ilecpp`, `msvc`, `pgcpp`, `sunpro`, and `unknown`), and `version` (the compiler version). On HP aCC compilers, `compiler` is instead the plain string `hp`. | +| `compiler` | Information on the used compiler. It is an object with the following keys: `c++` (the used C++ standard), `family` (the compiler family; possible values are `clang`, `icc`, `gcc`, `hp`, `ilecpp`, `msvc`, `pgcpp`, `sunpro`, and `unknown`), and `version` (the compiler version). | | `copyright` | The copyright line for the library as string. | | `name` | The name of the library as string. | | `platform` | The used platform as string. Possible values are `win32`, `linux`, `apple`, `unix`, and `unknown`. | diff --git a/include/nlohmann/detail/input/input_adapters.hpp b/include/nlohmann/detail/input/input_adapters.hpp index 09f0ded14..7f2f79cbf 100644 --- a/include/nlohmann/detail/input/input_adapters.hpp +++ b/include/nlohmann/detail/input/input_adapters.hpp @@ -746,7 +746,7 @@ struct container_input_adapter_factory< ContainerType, { // container is forwarded twice on purpose: the resulting begin/end // iterator types must match adapter_type, computed the same way - // NOLINTNEXTLINE(bugprone-use-after-move) + // NOLINTNEXTLINE(bugprone-use-after-move,hicpp-invalid-access-moved) return input_adapter(begin(std::forward(container)), end(std::forward(container))); } }; diff --git a/include/nlohmann/detail/string_concat.hpp b/include/nlohmann/detail/string_concat.hpp index a39dd5e63..74002c878 100644 --- a/include/nlohmann/detail/string_concat.hpp +++ b/include/nlohmann/detail/string_concat.hpp @@ -39,7 +39,6 @@ inline std::size_t concat_length(const char /*c*/, const Args& ... rest) template inline std::size_t concat_length(const char* cstr, const Args& ... rest) { - // cppcheck-suppress ignoredReturnValue return ::strlen(cstr) + concat_length(rest...); } diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 87e1dbc04..89ba13233 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -115,11 +115,12 @@ struct is_std_optional> : std::true_type {}; @brief a class to store JSON values @internal -@invariant The member variables @a m_value and @a m_type have the following -relationship: -- If `m_type == value_t::object`, then `m_value.object != nullptr`. -- If `m_type == value_t::array`, then `m_value.array != nullptr`. -- If `m_type == value_t::string`, then `m_value.string != nullptr`. +@invariant The member variables @a m_data.m_value and @a m_data.m_type have +the following relationship: +- If `m_data.m_type == value_t::object`, then `m_data.m_value.object != nullptr`. +- If `m_data.m_type == value_t::array`, then `m_data.m_value.array != nullptr`. +- If `m_data.m_type == value_t::string`, then `m_data.m_value.string != nullptr`. +- If `m_data.m_type == value_t::binary`, then `m_data.m_value.binary != nullptr`. The invariants are checked by member function assert_invariant(). @note ObjectType trick from https://stackoverflow.com/a/9860911 @@ -182,18 +183,12 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec } private: - using primitive_iterator_t = ::nlohmann::detail::primitive_iterator_t; - template - using internal_iterator = ::nlohmann::detail::internal_iterator; template using iter_impl = ::nlohmann::detail::iter_impl; template using iteration_proxy = ::nlohmann::detail::iteration_proxy; template using json_reverse_iterator = ::nlohmann::detail::json_reverse_iterator; - template - using output_adapter_t = ::nlohmann::detail::output_adapter_t; - template using binary_reader = ::nlohmann::detail::binary_reader; template using binary_writer = ::nlohmann::detail::binary_writer; @@ -334,8 +329,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec std::to_string(__GNUC_PATCHLEVEL__)) } }; -#elif defined(__HP_cc) || defined(__HP_aCC) - result["compiler"] = "hp" +#elif defined(__HP_aCC) + result["compiler"] = {{"family", "hp"}, {"version", __HP_aCC}}; #elif defined(__IBMCPP__) result["compiler"] = {{"family", "ilecpp"}, {"version", __IBMCPP__}}; #elif defined(_MSC_VER) @@ -477,9 +472,9 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec binary | binary | pointer to @ref binary_t null | null | *no value is stored* - @note Variable-length types (objects, arrays, and strings) are stored as - pointers. The size of the union should not exceed 64 bits if the default - value types are used. + @note Variable-length types (objects, arrays, strings, and binary + values) are stored as pointers. The size of the union should not exceed + 64 bits if the default value types are used. @since version 1.0.0 */ @@ -731,7 +726,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec end of every constructor to make sure that created objects respect the invariant. Furthermore, it has to be called each time the type of a JSON value is changed, because the invariant expresses a relationship between - @a m_type and @a m_value. + @a m_data.m_type and @a m_data.m_value. Furthermore, the parent relation is checked for arrays and objects: If @a check_parents true and the value is an array or object, then the @@ -752,7 +747,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec #if JSON_DIAGNOSTICS JSON_TRY { - // cppcheck-suppress assertWithSideEffect JSON_ASSERT(!check_parents || !is_structured() || std::all_of(begin(), end(), [this](const basic_json & j) { return j.m_parent == this; @@ -1924,8 +1918,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief create a null object /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ - basic_json(std::nullptr_t = nullptr) noexcept // NOLINT(bugprone-exception-escape) - : basic_json(value_t::null) + basic_json(std::nullptr_t = nullptr) noexcept { assert_invariant(); } @@ -2289,15 +2282,15 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief move constructor /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ basic_json(basic_json&& other) noexcept - : json_base_class_t(std::forward(other)), - m_data(std::move(other.m_data)) // cppcheck-suppress[accessForwarded] TODO check + : json_base_class_t(std::move(static_cast(other))), + m_data(std::move(other.m_data)) #if JSON_DIAGNOSTIC_POSITIONS - , start_position(other.start_position) // cppcheck-suppress[accessForwarded] TODO check - , end_position(other.end_position) // cppcheck-suppress[accessForwarded] TODO check + , start_position(other.start_position) + , end_position(other.end_position) #endif { // check that the passed value is valid - other.assert_invariant(false); // cppcheck-suppress[accessForwarded] + other.assert_invariant(false); // invalidate payload other.m_data.m_type = value_t::null; @@ -2864,7 +2857,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec @tparam PointerType pointer type; must be a pointer to @ref array_t, @ref object_t, @ref string_t, @ref boolean_t, @ref number_integer_t, - @ref number_unsigned_t, or @ref number_float_t. + @ref number_unsigned_t, @ref number_float_t, or @ref binary_t. @return pointer to the internally stored JSON value if the requested pointer type @a PointerType fits to the JSON value; `nullptr` otherwise @@ -3032,6 +3025,90 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @} + private: + /// @brief look up @a key in the object held by @a j, for either constness of @a j + /// @note the single place that performs a (possibly transparent) object key lookup + template + static auto object_lookup(Self& j, KeyType&& key) + -> decltype(j.m_data.m_value.object->find(lookup_key(std::forward(key)))) + { + return j.m_data.m_value.object->find(lookup_key(std::forward(key))); + } + + /// @brief checked object element access used by the at() overloads taking a key + /// @throw type_error.304 if @a j is not an object + /// @throw out_of_range.403 if @a key is not found + template + static auto object_at(Self& j, KeyType&& key) + -> decltype((object_lookup(j, std::forward(key))->second)) + { + // at only works for objects + if (JSON_HEDLEY_UNLIKELY(!j.is_object())) + { + JSON_THROW(type_error::create(304, detail::concat("cannot use at() with ", j.type_name()), &j)); + } + + auto it = object_lookup(j, std::forward(key)); + if (it == j.m_data.m_value.object->end()) + { + // key is only forwarded into the lookup above: object_t::find() (a plain + // std::map or ordered_map) never moves from its argument, so key is still + // valid here regardless of whether KeyType was deduced as an rvalue reference + // NOLINTNEXTLINE(bugprone-use-after-move,hicpp-invalid-access-moved) + JSON_THROW(out_of_range::create(403, detail::concat("key '", string_t(key), "' not found"), &j)); + } + return it->second; + } + + /// @brief checked array element access used by the at() overloads taking an index + /// @throw type_error.304 if @a j is not an array + /// @throw out_of_range.401 if @a idx is out of range + template + static auto array_at(Self& j, size_type idx) + -> decltype((*j.m_data.m_value.array)[idx]) + { + // at only works for arrays + if (JSON_HEDLEY_UNLIKELY(!j.is_array())) + { + JSON_THROW(type_error::create(304, detail::concat("cannot use at() with ", j.type_name()), &j)); + } + + if (JSON_HEDLEY_UNLIKELY(idx >= j.m_data.m_value.array->size())) + { + JSON_THROW(out_of_range::create(401, detail::concat("array index ", std::to_string(idx), " is out of range"), &j)); + } + + return (*j.m_data.m_value.array)[idx]; + } + + /// @brief convert a null value to an empty container of type @a Container + /// @tparam Container array_t or object_t; any other type does not compile + template + void convert_null_to() + { + JSON_ASSERT(is_null()); + // create the container before touching the type, so a throwing + // allocation leaves this value as a valid null rather than a type + // tag with a dangling/null pointer behind it + set_container(create()); + assert_invariant(); + } + + /// @brief store a freshly created array and set the matching type + void set_container(array_t* array) noexcept + { + m_data.m_value.array = array; + m_data.m_type = value_t::array; + } + + /// @brief store a freshly created object and set the matching type + void set_container(object_t* object) noexcept + { + m_data.m_value.object = object; + m_data.m_type = value_t::object; + } + + public: //////////////////// // element access // //////////////////// @@ -3044,54 +3121,21 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/at/ reference at(size_type idx) { - // at only works for arrays - if (JSON_HEDLEY_UNLIKELY(!is_array())) - { - JSON_THROW(type_error::create(304, detail::concat("cannot use at() with ", type_name()), this)); - } - - if (JSON_HEDLEY_UNLIKELY(idx >= m_data.m_value.array->size())) - { - JSON_THROW(out_of_range::create(401, detail::concat("array index ", std::to_string(idx), " is out of range"), this)); - } - - return set_parent((*m_data.m_value.array)[idx]); + return set_parent(array_at(*this, idx)); } /// @brief access specified array element with bounds checking /// @sa https://json.nlohmann.me/api/basic_json/at/ const_reference at(size_type idx) const { - // at only works for arrays - if (JSON_HEDLEY_UNLIKELY(!is_array())) - { - JSON_THROW(type_error::create(304, detail::concat("cannot use at() with ", type_name()), this)); - } - - if (JSON_HEDLEY_UNLIKELY(idx >= m_data.m_value.array->size())) - { - JSON_THROW(out_of_range::create(401, detail::concat("array index ", std::to_string(idx), " is out of range"), this)); - } - - return (*m_data.m_value.array)[idx]; + return array_at(*this, idx); } /// @brief access specified object element with bounds checking /// @sa https://json.nlohmann.me/api/basic_json/at/ reference at(const typename object_t::key_type& key) { - // at only works for objects - if (JSON_HEDLEY_UNLIKELY(!is_object())) - { - JSON_THROW(type_error::create(304, detail::concat("cannot use at() with ", type_name()), this)); - } - - auto it = m_data.m_value.object->find(key); - if (it == m_data.m_value.object->end()) - { - JSON_THROW(out_of_range::create(403, detail::concat("key '", key, "' not found"), this)); - } - return set_parent(it->second); + return set_parent(object_at(*this, key)); } /// @brief access specified object element with bounds checking @@ -3100,36 +3144,14 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec detail::is_usable_as_basic_json_key_type::value, int> = 0> reference at(KeyType && key) { - // at only works for objects - if (JSON_HEDLEY_UNLIKELY(!is_object())) - { - JSON_THROW(type_error::create(304, detail::concat("cannot use at() with ", type_name()), this)); - } - - auto it = m_data.m_value.object->find(lookup_key(std::forward(key))); - if (it == m_data.m_value.object->end()) - { - JSON_THROW(out_of_range::create(403, detail::concat("key '", string_t(std::forward(key)), "' not found"), this)); - } - return set_parent(it->second); + return set_parent(object_at(*this, std::forward(key))); } /// @brief access specified object element with bounds checking /// @sa https://json.nlohmann.me/api/basic_json/at/ const_reference at(const typename object_t::key_type& key) const { - // at only works for objects - if (JSON_HEDLEY_UNLIKELY(!is_object())) - { - JSON_THROW(type_error::create(304, detail::concat("cannot use at() with ", type_name()), this)); - } - - auto it = m_data.m_value.object->find(key); - if (it == m_data.m_value.object->end()) - { - JSON_THROW(out_of_range::create(403, detail::concat("key '", key, "' not found"), this)); - } - return it->second; + return object_at(*this, key); } /// @brief access specified object element with bounds checking @@ -3138,18 +3160,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec detail::is_usable_as_basic_json_key_type::value, int> = 0> const_reference at(KeyType && key) const { - // at only works for objects - if (JSON_HEDLEY_UNLIKELY(!is_object())) - { - JSON_THROW(type_error::create(304, detail::concat("cannot use at() with ", type_name()), this)); - } - - auto it = m_data.m_value.object->find(lookup_key(std::forward(key))); - if (it == m_data.m_value.object->end()) - { - JSON_THROW(out_of_range::create(403, detail::concat("key '", string_t(std::forward(key)), "' not found"), this)); - } - return it->second; + return object_at(*this, std::forward(key)); } /// @brief access specified array element @@ -3159,9 +3170,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // implicitly convert a null value to an empty array if (is_null()) { - m_data.m_type = value_t::array; - m_data.m_value.array = create(); - assert_invariant(); + convert_null_to(); } // operator[] only works for arrays @@ -3227,9 +3236,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // implicitly convert a null value to an empty object if (is_null()) { - m_data.m_type = value_t::object; - m_data.m_value.object = create(); - assert_invariant(); + convert_null_to(); } // operator[] only works for objects @@ -3249,7 +3256,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // const operator[] only works for objects if (JSON_HEDLEY_LIKELY(is_object())) { - auto it = m_data.m_value.object->find(key); + auto it = object_lookup(*this, key); JSON_ASSERT(it != m_data.m_value.object->end()); return it->second; } @@ -3280,9 +3287,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // implicitly convert a null value to an empty object if (is_null()) { - m_data.m_type = value_t::object; - m_data.m_value.object = create(); - assert_invariant(); + convert_null_to(); } // operator[] only works for objects @@ -3304,7 +3309,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // const operator[] only works for objects if (JSON_HEDLEY_LIKELY(is_object())) { - auto it = m_data.m_value.object->find(lookup_key(std::forward(key))); + auto it = object_lookup(*this, std::forward(key)); JSON_ASSERT(it != m_data.m_value.object->end()); return it->second; } @@ -3324,6 +3329,36 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec detail::is_c_string_uncvref::value, string_t, typename std::decay::type >; + /// @brief look up @a key for value(), the single place shared by all key-based overloads + /// @throw type_error.306 if this is not an object + /// @return a pointer to the found value, or `nullptr` if @a key was not found + template + const basic_json* value_member(KeyType&& key) const + { + // value only works for objects + if (JSON_HEDLEY_UNLIKELY(!is_object())) + { + JSON_THROW(type_error::create(306, detail::concat("cannot use value() with ", type_name()), this)); + } + + const auto it = find(std::forward(key)); + return it != end() ? &*it : nullptr; + } + + /// @brief resolve @a ptr for value(), the single place shared by both json_pointer overloads + /// @throw type_error.306 if this is not an array or object + /// @return a pointer to the resolved value, or `nullptr` if @a ptr does not resolve + const basic_json* value_pointee(const json_pointer& ptr) const + { + // value only works for arrays and objects + if (JSON_HEDLEY_UNLIKELY(!is_structured())) + { + JSON_THROW(type_error::create(306, detail::concat("cannot use value() with ", type_name()), this)); + } + + return ptr.get_checked_or_null(this); + } + public: // an integer literal 0 would otherwise convert to a null const char* and from there to key_type template::value, int> = 0> @@ -3337,20 +3372,9 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec && !std::is_same>::value, int > = 0 > ValueType value(const typename object_t::key_type& key, const ValueType& default_value) const { - // value only works for objects - if (JSON_HEDLEY_LIKELY(is_object())) - { - // If 'key' is found, return its value. Otherwise, return `default_value'. - const auto it = find(key); - if (it != end()) - { - return it->template get(); - } - - return default_value; - } - - JSON_THROW(type_error::create(306, detail::concat("cannot use value() with ", type_name()), this)); + // If 'key' is found, return its value. Otherwise, return `default_value'. + const auto* found = value_member(key); + return found != nullptr ? found->template get() : default_value; } /// @brief access specified object element with default value @@ -3362,20 +3386,9 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec && !std::is_same>::value, int > = 0 > ReturnType value(const typename object_t::key_type& key, ValueType && default_value) const { - // value only works for objects - if (JSON_HEDLEY_LIKELY(is_object())) - { - // If 'key' is found, return its value. Otherwise, return `default_value'. - const auto it = find(key); - if (it != end()) - { - return it->template get(); - } - - return std::forward(default_value); - } - - JSON_THROW(type_error::create(306, detail::concat("cannot use value() with ", type_name()), this)); + // If 'key' is found, return its value. Otherwise, return `default_value'. + const auto* found = value_member(key); + return found != nullptr ? found->template get() : std::forward(default_value); } /// @brief access specified object element with default value @@ -3388,23 +3401,12 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec && !std::is_same>::value, int > = 0 > ValueType value(KeyType && key, const ValueType& default_value) const { - // value only works for objects - if (JSON_HEDLEY_LIKELY(is_object())) - { - // If 'key' is found, return its value. Otherwise, return `default_value'. - const auto it = find(std::forward(key)); - if (it != end()) - { - return it->template get(); - } - - return default_value; - } - - JSON_THROW(type_error::create(306, detail::concat("cannot use value() with ", type_name()), this)); + // If 'key' is found, return its value. Otherwise, return `default_value'. + const auto* found = value_member(std::forward(key)); + return found != nullptr ? found->template get() : default_value; } - /// @brief access specified object element via JSON Pointer with default value + /// @brief access specified object element with default value /// @sa https://json.nlohmann.me/api/basic_json/value/ template < class ValueType, class KeyType, class ReturnType = typename value_return_type::type, detail::enable_if_t < @@ -3415,20 +3417,9 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec && !std::is_same>::value, int > = 0 > ReturnType value(KeyType && key, ValueType && default_value) const { - // value only works for objects - if (JSON_HEDLEY_LIKELY(is_object())) - { - // If 'key' is found, return its value. Otherwise, return `default_value'. - const auto it = find(std::forward(key)); - if (it != end()) - { - return it->template get(); - } - - return std::forward(default_value); - } - - JSON_THROW(type_error::create(306, detail::concat("cannot use value() with ", type_name()), this)); + // If 'key' is found, return its value. Otherwise, return `default_value'. + const auto* found = value_member(std::forward(key)); + return found != nullptr ? found->template get() : std::forward(default_value); } /// @brief access specified object element via JSON Pointer with default value @@ -3438,21 +3429,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec && !std::is_same>::value, int > = 0 > ValueType value(const json_pointer& ptr, const ValueType& default_value) const { - // value only works for arrays and objects - if (JSON_HEDLEY_LIKELY(is_structured())) - { - // If the pointer resolves to a value, return it. Otherwise, return - // 'default_value'. - const auto* res = ptr.get_checked_or_null(this); - if (JSON_HEDLEY_LIKELY(res != nullptr)) - { - return res->template get(); - } - - return default_value; - } - - JSON_THROW(type_error::create(306, detail::concat("cannot use value() with ", type_name()), this)); + // If the pointer resolves to a value, return it. Otherwise, return + // 'default_value'. + const auto* found = value_pointee(ptr); + return found != nullptr ? found->template get() : default_value; } /// @brief access specified object element via JSON Pointer with default value @@ -3463,21 +3443,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec && !std::is_same>::value, int > = 0 > ReturnType value(const json_pointer& ptr, ValueType && default_value) const { - // value only works for arrays and objects - if (JSON_HEDLEY_LIKELY(is_structured())) - { - // If the pointer resolves to a value, return it. Otherwise, return - // 'default_value'. - const auto* res = ptr.get_checked_or_null(this); - if (JSON_HEDLEY_LIKELY(res != nullptr)) - { - return res->template get(); - } - - return std::forward(default_value); - } - - JSON_THROW(type_error::create(306, detail::concat("cannot use value() with ", type_name()), this)); + // If the pointer resolves to a value, return it. Otherwise, return + // 'default_value'. + const auto* found = value_pointee(ptr); + return found != nullptr ? found->template get() : std::forward(default_value); } template < class ValueType, class BasicJsonType, detail::enable_if_t < @@ -3562,21 +3531,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec JSON_THROW(invalid_iterator::create(205, "iterator out of range", this)); } - if (is_string()) - { - AllocatorType alloc; - std::allocator_traits::destroy(alloc, m_data.m_value.string); - std::allocator_traits::deallocate(alloc, m_data.m_value.string, 1); - m_data.m_value.string = nullptr; - } - else if (is_binary()) - { - AllocatorType alloc; - std::allocator_traits::destroy(alloc, m_data.m_value.binary); - std::allocator_traits::deallocate(alloc, m_data.m_value.binary, 1); - m_data.m_value.binary = nullptr; - } - + m_data.m_value.destroy(m_data.m_type); + m_data.m_value = {}; m_data.m_type = value_t::null; assert_invariant(); break; @@ -3628,27 +3584,14 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec case value_t::string: case value_t::binary: { - if (JSON_HEDLEY_LIKELY(!first.m_it.primitive_iterator.is_begin() - || !last.m_it.primitive_iterator.is_end())) + if (JSON_HEDLEY_UNLIKELY(!first.m_it.primitive_iterator.is_begin() + || !last.m_it.primitive_iterator.is_end())) { JSON_THROW(invalid_iterator::create(204, "iterators out of range", this)); } - if (is_string()) - { - AllocatorType alloc; - std::allocator_traits::destroy(alloc, m_data.m_value.string); - std::allocator_traits::deallocate(alloc, m_data.m_value.string, 1); - m_data.m_value.string = nullptr; - } - else if (is_binary()) - { - AllocatorType alloc; - std::allocator_traits::destroy(alloc, m_data.m_value.binary); - std::allocator_traits::deallocate(alloc, m_data.m_value.binary, 1); - m_data.m_value.binary = nullptr; - } - + m_data.m_value.destroy(m_data.m_type); + m_data.m_value = {}; m_data.m_type = value_t::null; assert_invariant(); break; @@ -3704,7 +3647,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec JSON_THROW(type_error::create(307, detail::concat("cannot use erase() with ", type_name()), this)); } - const auto it = m_data.m_value.object->find(lookup_key(std::forward(key))); + const auto it = object_lookup(*this, std::forward(key)); if (it != m_data.m_value.object->end()) { m_data.m_value.object->erase(it); @@ -3781,7 +3724,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec if (is_object()) { - result.m_it.object_iterator = m_data.m_value.object->find(key); + result.m_it.object_iterator = object_lookup(*this, key); } return result; @@ -3795,7 +3738,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec if (is_object()) { - result.m_it.object_iterator = m_data.m_value.object->find(key); + result.m_it.object_iterator = object_lookup(*this, key); } return result; @@ -3811,7 +3754,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec if (is_object()) { - result.m_it.object_iterator = m_data.m_value.object->find(lookup_key(std::forward(key))); + result.m_it.object_iterator = object_lookup(*this, std::forward(key)); } return result; @@ -3827,7 +3770,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec if (is_object()) { - result.m_it.object_iterator = m_data.m_value.object->find(lookup_key(std::forward(key))); + result.m_it.object_iterator = object_lookup(*this, std::forward(key)); } return result; @@ -3858,7 +3801,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec JSON_HEDLEY_WARN_UNUSED_RESULT bool contains(const typename object_t::key_type& key) const { - return is_object() && m_data.m_value.object->find(key) != m_data.m_value.object->end(); + return is_object() && object_lookup(*this, key) != m_data.m_value.object->end(); } /// @brief check the existence of an element in a JSON object @@ -3868,7 +3811,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec JSON_HEDLEY_WARN_UNUSED_RESULT bool contains(KeyType && key) const { - return is_object() && m_data.m_value.object->find(lookup_key(std::forward(key))) != m_data.m_value.object->end(); + return is_object() && object_lookup(*this, std::forward(key)) != m_data.m_value.object->end(); } /// @brief check the existence of an element in a JSON object given a JSON pointer @@ -4233,9 +4176,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // transform a null object into an array if (is_null()) { - m_data.m_type = value_t::array; - m_data.m_value = value_t::array; - assert_invariant(); + convert_null_to(); } // add the element to the array (move semantics) @@ -4266,9 +4207,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // transform a null object into an array if (is_null()) { - m_data.m_type = value_t::array; - m_data.m_value = value_t::array; - assert_invariant(); + convert_null_to(); } // add the element to the array @@ -4298,9 +4237,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // transform a null object into an object if (is_null()) { - m_data.m_type = value_t::object; - m_data.m_value = value_t::object; - assert_invariant(); + convert_null_to(); } // add the element to the object @@ -4354,9 +4291,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // transform a null object into an array if (is_null()) { - m_data.m_type = value_t::array; - m_data.m_value = value_t::array; - assert_invariant(); + convert_null_to(); } // add the element to the array (perfect forwarding) @@ -4379,9 +4314,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // transform a null object into an object if (is_null()) { - m_data.m_type = value_t::object; - m_data.m_value = value_t::object; - assert_invariant(); + convert_null_to(); } // add the element to the array (perfect forwarding) @@ -4441,7 +4374,22 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/insert/ iterator insert(const_iterator pos, basic_json&& val) // NOLINT(performance-unnecessary-value-param) { - return insert(std::move(pos), val); + // insert only works for arrays + if (JSON_HEDLEY_LIKELY(is_array())) + { + // check if iterator pos fits to this JSON value + if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) + { + JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value", this)); + } + + // moving into a local first keeps this safe even if val aliases + // an element of this array + basic_json tmp(std::move(val)); + return insert_iterator(pos, std::move(tmp)); + } + + JSON_THROW(type_error::create(309, detail::concat("cannot use insert() with ", type_name()), this)); } /// @brief inserts copies of element into array @@ -4617,14 +4565,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// value is an object; called first by both @ref update overloads void prepare_update() { - // implicitly convert a null value to an empty object; create the - // object before setting the type, so a throwing allocation leaves - // this value null + // implicitly convert a null value to an empty object if (is_null()) { - m_data.m_value.object = create(); - m_data.m_type = value_t::object; - assert_invariant(); + convert_null_to(); } if (JSON_HEDLEY_UNLIKELY(!is_object())) @@ -4833,7 +4777,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/swap/ void swap(binary_t& other) // NOLINT(bugprone-exception-escape,cppcoreguidelines-noexcept-swap,performance-noexcept-swap) { - // swap only works for strings + // swap only works for binary values if (JSON_HEDLEY_LIKELY(is_binary())) { using std::swap; @@ -4849,7 +4793,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/swap/ void swap(typename binary_t::container_type& other) // NOLINT(bugprone-exception-escape) { - // swap only works for strings + // swap only works for binary values if (JSON_HEDLEY_LIKELY(is_binary())) { using std::swap; @@ -5350,7 +5294,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool ignore_trailing_commas = false) { basic_json result; - parser(detail::input_adapter(std::forward(i)), std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas).parse(true, result); // cppcheck-suppress[accessMoved,accessForwarded] + auto p = parser(detail::input_adapter(std::forward(i)), std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas); + p.parse(true, result); return result; } @@ -5367,7 +5312,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool ignore_trailing_commas = false) { basic_json result; - parser(detail::input_adapter(std::move(first), std::move(last)), std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas).parse(true, result); // cppcheck-suppress[accessMoved] + auto p = parser(detail::input_adapter(std::move(first), std::move(last)), std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas); + p.parse(true, result); return result; } @@ -5380,7 +5326,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool ignore_trailing_commas = false) { basic_json result; - parser(i.get(), std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas).parse(true, result); // cppcheck-suppress[accessMoved] + auto p = parser(i.get(), std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas); + p.parse(true, result); return result; } @@ -5607,6 +5554,31 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec } #endif + private: + /*! + @brief shared implementation of the binary from_cbor/from_msgpack/ + from_ubjson/from_bjdata/from_bon8/from_bson overloads + + Building the @ref binary_reader as a named local, rather than as a + temporary that @ref detail::json_sax_dom_parser::parse is called on in + the same expression, avoids a false-positive cppcheck accessMoved + warning in each of the 16 callers. + */ + template + static basic_json from_binary_impl(InputAdapterType ia, const input_format_t format, + const bool strict, const bool allow_exceptions, + const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + binary_reader reader(std::move(ia), format); + if (!reader.sax_parse(&sdp, strict, tag_handler)) + { + result = value_t::discarded; + } + return result; + } + ////////////////////////////////////////// // binary serialization/deserialization // ////////////////////////////////////////// @@ -5779,14 +5751,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool allow_exceptions = true, const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error) { - basic_json result; - auto ia = detail::input_adapter(std::forward(i)); - detail::json_sax_dom_parser sdp(result, allow_exceptions); - if (!binary_reader(std::move(ia), input_format_t::cbor).sax_parse(&sdp, strict, tag_handler)) // cppcheck-suppress[accessMoved] - { - result = value_t::discarded; - } - return result; + return from_binary_impl(detail::input_adapter(std::forward(i)), input_format_t::cbor, strict, allow_exceptions, tag_handler); } /// @brief create a JSON value from an input in CBOR format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -5799,14 +5764,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool allow_exceptions = true, const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error) { - basic_json result; - auto ia = detail::input_adapter(std::move(first), std::move(last)); - detail::json_sax_dom_parser sdp(result, allow_exceptions); - if (!binary_reader(std::move(ia), input_format_t::cbor).sax_parse(&sdp, strict, tag_handler)) // cppcheck-suppress[accessMoved] - { - result = value_t::discarded; - } - return result; + return from_binary_impl(detail::input_adapter(std::move(first), std::move(last)), input_format_t::cbor, strict, allow_exceptions, tag_handler); } template @@ -5827,15 +5785,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool allow_exceptions = true, const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error) { - basic_json result; - auto ia = i.get(); - detail::json_sax_dom_parser sdp(result, allow_exceptions); - // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) - if (!binary_reader(std::move(ia), input_format_t::cbor).sax_parse(&sdp, strict, tag_handler)) // cppcheck-suppress[accessMoved] - { - result = value_t::discarded; - } - return result; + return from_binary_impl(i.get(), input_format_t::cbor, strict, allow_exceptions, tag_handler); } /// @brief create a JSON value from an input in MessagePack format @@ -5846,14 +5796,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool strict = true, const bool allow_exceptions = true) { - basic_json result; - auto ia = detail::input_adapter(std::forward(i)); - detail::json_sax_dom_parser sdp(result, allow_exceptions); - if (!binary_reader(std::move(ia), input_format_t::msgpack).sax_parse(&sdp, strict)) // cppcheck-suppress[accessMoved] - { - result = value_t::discarded; - } - return result; + return from_binary_impl(detail::input_adapter(std::forward(i)), input_format_t::msgpack, strict, allow_exceptions); } /// @brief create a JSON value from an input in MessagePack format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -5865,14 +5808,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool strict = true, const bool allow_exceptions = true) { - basic_json result; - auto ia = detail::input_adapter(std::move(first), std::move(last)); - detail::json_sax_dom_parser sdp(result, allow_exceptions); - if (!binary_reader(std::move(ia), input_format_t::msgpack).sax_parse(&sdp, strict)) // cppcheck-suppress[accessMoved] - { - result = value_t::discarded; - } - return result; + return from_binary_impl(detail::input_adapter(std::move(first), std::move(last)), input_format_t::msgpack, strict, allow_exceptions); } template @@ -5891,15 +5827,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool strict = true, const bool allow_exceptions = true) { - basic_json result; - auto ia = i.get(); - detail::json_sax_dom_parser sdp(result, allow_exceptions); - // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) - if (!binary_reader(std::move(ia), input_format_t::msgpack).sax_parse(&sdp, strict)) // cppcheck-suppress[accessMoved] - { - result = value_t::discarded; - } - return result; + return from_binary_impl(i.get(), input_format_t::msgpack, strict, allow_exceptions); } /// @brief create a JSON value from an input in UBJSON format @@ -5910,14 +5838,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool strict = true, const bool allow_exceptions = true) { - basic_json result; - auto ia = detail::input_adapter(std::forward(i)); - detail::json_sax_dom_parser sdp(result, allow_exceptions); - if (!binary_reader(std::move(ia), input_format_t::ubjson).sax_parse(&sdp, strict)) // cppcheck-suppress[accessMoved] - { - result = value_t::discarded; - } - return result; + return from_binary_impl(detail::input_adapter(std::forward(i)), input_format_t::ubjson, strict, allow_exceptions); } /// @brief create a JSON value from an input in UBJSON format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -5929,14 +5850,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool strict = true, const bool allow_exceptions = true) { - basic_json result; - auto ia = detail::input_adapter(std::move(first), std::move(last)); - detail::json_sax_dom_parser sdp(result, allow_exceptions); - if (!binary_reader(std::move(ia), input_format_t::ubjson).sax_parse(&sdp, strict)) // cppcheck-suppress[accessMoved] - { - result = value_t::discarded; - } - return result; + return from_binary_impl(detail::input_adapter(std::move(first), std::move(last)), input_format_t::ubjson, strict, allow_exceptions); } template @@ -5955,15 +5869,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool strict = true, const bool allow_exceptions = true) { - basic_json result; - auto ia = i.get(); - detail::json_sax_dom_parser sdp(result, allow_exceptions); - // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) - if (!binary_reader(std::move(ia), input_format_t::ubjson).sax_parse(&sdp, strict)) // cppcheck-suppress[accessMoved] - { - result = value_t::discarded; - } - return result; + return from_binary_impl(i.get(), input_format_t::ubjson, strict, allow_exceptions); } /// @brief create a JSON value from an input in BJData format @@ -5974,14 +5880,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool strict = true, const bool allow_exceptions = true) { - basic_json result; - auto ia = detail::input_adapter(std::forward(i)); - detail::json_sax_dom_parser sdp(result, allow_exceptions); - if (!binary_reader(std::move(ia), input_format_t::bjdata).sax_parse(&sdp, strict)) // cppcheck-suppress[accessMoved] - { - result = value_t::discarded; - } - return result; + return from_binary_impl(detail::input_adapter(std::forward(i)), input_format_t::bjdata, strict, allow_exceptions); } /// @brief create a JSON value from an input in BJData format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -5993,14 +5892,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool strict = true, const bool allow_exceptions = true) { - basic_json result; - auto ia = detail::input_adapter(std::move(first), std::move(last)); - detail::json_sax_dom_parser sdp(result, allow_exceptions); - if (!binary_reader(std::move(ia), input_format_t::bjdata).sax_parse(&sdp, strict)) // cppcheck-suppress[accessMoved] - { - result = value_t::discarded; - } - return result; + return from_binary_impl(detail::input_adapter(std::move(first), std::move(last)), input_format_t::bjdata, strict, allow_exceptions); } /// @brief create a JSON value from an input in BON8 format @@ -6011,14 +5903,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool strict = true, const bool allow_exceptions = true) { - basic_json result; - auto ia = detail::input_adapter(std::forward(i)); - detail::json_sax_dom_parser sdp(result, allow_exceptions); - if (!binary_reader(std::move(ia), input_format_t::bon8).sax_parse(&sdp, strict)) // cppcheck-suppress[accessMoved] - { - result = value_t::discarded; - } - return result; + return from_binary_impl(detail::input_adapter(std::forward(i)), input_format_t::bon8, strict, allow_exceptions); } /// @brief create a JSON value from an input in BON8 format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -6030,14 +5915,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool strict = true, const bool allow_exceptions = true) { - basic_json result; - auto ia = detail::input_adapter(std::move(first), std::move(last)); - detail::json_sax_dom_parser sdp(result, allow_exceptions); - if (!binary_reader(std::move(ia), input_format_t::bon8).sax_parse(&sdp, strict)) // cppcheck-suppress[accessMoved] - { - result = value_t::discarded; - } - return result; + return from_binary_impl(detail::input_adapter(std::move(first), std::move(last)), input_format_t::bon8, strict, allow_exceptions); } /// @brief create a JSON value from an input in BSON format @@ -6048,14 +5926,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool strict = true, const bool allow_exceptions = true) { - basic_json result; - auto ia = detail::input_adapter(std::forward(i)); - detail::json_sax_dom_parser sdp(result, allow_exceptions); - if (!binary_reader(std::move(ia), input_format_t::bson).sax_parse(&sdp, strict)) // cppcheck-suppress[accessMoved] - { - result = value_t::discarded; - } - return result; + return from_binary_impl(detail::input_adapter(std::forward(i)), input_format_t::bson, strict, allow_exceptions); } /// @brief create a JSON value from an input in BSON format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -6067,14 +5938,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool strict = true, const bool allow_exceptions = true) { - basic_json result; - auto ia = detail::input_adapter(std::move(first), std::move(last)); - detail::json_sax_dom_parser sdp(result, allow_exceptions); - if (!binary_reader(std::move(ia), input_format_t::bson).sax_parse(&sdp, strict)) // cppcheck-suppress[accessMoved] - { - result = value_t::discarded; - } - return result; + return from_binary_impl(detail::input_adapter(std::move(first), std::move(last)), input_format_t::bson, strict, allow_exceptions); } template @@ -6093,15 +5957,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool strict = true, const bool allow_exceptions = true) { - basic_json result; - auto ia = i.get(); - detail::json_sax_dom_parser sdp(result, allow_exceptions); - // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) - if (!binary_reader(std::move(ia), input_format_t::bson).sax_parse(&sdp, strict)) // cppcheck-suppress[accessMoved] - { - result = value_t::discarded; - } - return result; + return from_binary_impl(i.get(), input_format_t::bson, strict, allow_exceptions); } /// @} diff --git a/include/nlohmann/ordered_map.hpp b/include/nlohmann/ordered_map.hpp index d1c247483..eb521114e 100644 --- a/include/nlohmann/ordered_map.hpp +++ b/include/nlohmann/ordered_map.hpp @@ -74,16 +74,43 @@ template , return *this; } +private: + /// @brief find the entry for @a key, for either constness of @a self + /// @note the single place that performs the linear key search + template + static auto find_impl(Self& self, KeyType&& key) -> decltype(self.begin()) + { + for (auto it = self.begin(); it != self.end(); ++it) + { + if (self.m_compare(it->first, key)) + { + return it; + } + } + return self.end(); + } + + /// @brief remove the entry @a it points to, preserving order + /// @note keys are not movable, so the tail is destroyed and re-constructed in place + void erase_at(iterator it) + { + for (auto next = it; ++next != this->end(); ++it) + { + it->~value_type(); // Destroy but keep allocation + new (&*it) value_type{std::move(*next)}; + } + Container::pop_back(); + } + +public: template::value, int> = 0> std::pair emplace(const key_type& key, V && t) { - for (auto it = this->begin(); it != this->end(); ++it) + const auto it = find_impl(*this, key); + if (it != this->end()) { - if (m_compare(it->first, key)) - { - return {it, false}; - } + return {it, false}; } append(key, std::forward(t)); return {std::prev(this->end()), true}; @@ -94,12 +121,10 @@ template , detail::is_constructible>::value, int> = 0> std::pair emplace(KeyType && key, V && t) { - for (auto it = this->begin(); it != this->end(); ++it) + const auto it = find_impl(*this, key); + if (it != this->end()) { - if (m_compare(it->first, key)) - { - return {it, false}; - } + return {it, false}; } append(std::forward(key), std::forward(t)); return {std::prev(this->end()), true}; @@ -131,75 +156,55 @@ template , T& at(const key_type& key) { - for (auto it = this->begin(); it != this->end(); ++it) + const auto it = find_impl(*this, key); + if (it == this->end()) { - if (m_compare(it->first, key)) - { - return it->second; - } + JSON_THROW(std::out_of_range("key not found")); } - - JSON_THROW(std::out_of_range("key not found")); + return it->second; } template::value, int> = 0> T & at(KeyType && key) // NOLINT(cppcoreguidelines-missing-std-forward) { - for (auto it = this->begin(); it != this->end(); ++it) + const auto it = find_impl(*this, key); + if (it == this->end()) { - if (m_compare(it->first, key)) - { - return it->second; - } + JSON_THROW(std::out_of_range("key not found")); } - - JSON_THROW(std::out_of_range("key not found")); + return it->second; } const T& at(const key_type& key) const { - for (auto it = this->begin(); it != this->end(); ++it) + const auto it = find_impl(*this, key); + if (it == this->end()) { - if (m_compare(it->first, key)) - { - return it->second; - } + JSON_THROW(std::out_of_range("key not found")); } - - JSON_THROW(std::out_of_range("key not found")); + return it->second; } template::value, int> = 0> const T & at(KeyType && key) const // NOLINT(cppcoreguidelines-missing-std-forward) { - for (auto it = this->begin(); it != this->end(); ++it) + const auto it = find_impl(*this, key); + if (it == this->end()) { - if (m_compare(it->first, key)) - { - return it->second; - } + JSON_THROW(std::out_of_range("key not found")); } - - JSON_THROW(std::out_of_range("key not found")); + return it->second; } size_type erase(const key_type& key) { - for (auto it = this->begin(); it != this->end(); ++it) + const auto it = find_impl(*this, key); + if (it != this->end()) { - if (m_compare(it->first, key)) - { - // Since we cannot move const Keys, re-construct them in place - for (auto next = it; ++next != this->end(); ++it) - { - it->~value_type(); // Destroy but keep allocation - new (&*it) value_type{std::move(*next)}; - } - Container::pop_back(); - return 1; - } + erase_at(it); + return 1; } return 0; } @@ -208,19 +213,11 @@ template , detail::is_usable_as_key_type::value, int> = 0> size_type erase(KeyType && key) // NOLINT(cppcoreguidelines-missing-std-forward) { - for (auto it = this->begin(); it != this->end(); ++it) + const auto it = find_impl(*this, key); + if (it != this->end()) { - if (m_compare(it->first, key)) - { - // Since we cannot move const Keys, re-construct them in place - for (auto next = it; ++next != this->end(); ++it) - { - it->~value_type(); // Destroy but keep allocation - new (&*it) value_type{std::move(*next)}; - } - Container::pop_back(); - return 1; - } + erase_at(it); + return 1; } return 0; } @@ -285,80 +282,38 @@ template , size_type count(const key_type& key) const { - for (auto it = this->begin(); it != this->end(); ++it) - { - if (m_compare(it->first, key)) - { - return 1; - } - } - return 0; + return find_impl(*this, key) != this->end() ? 1 : 0; } template::value, int> = 0> size_type count(KeyType && key) const // NOLINT(cppcoreguidelines-missing-std-forward) { - for (auto it = this->begin(); it != this->end(); ++it) - { - if (m_compare(it->first, key)) - { - return 1; - } - } - return 0; + return find_impl(*this, key) != this->end() ? 1 : 0; } iterator find(const key_type& key) { - for (auto it = this->begin(); it != this->end(); ++it) - { - if (m_compare(it->first, key)) - { - return it; - } - } - return Container::end(); + return find_impl(*this, key); } template::value, int> = 0> iterator find(KeyType && key) // NOLINT(cppcoreguidelines-missing-std-forward) { - for (auto it = this->begin(); it != this->end(); ++it) - { - if (m_compare(it->first, key)) - { - return it; - } - } - return Container::end(); + return find_impl(*this, key); } const_iterator find(const key_type& key) const { - for (auto it = this->begin(); it != this->end(); ++it) - { - if (m_compare(it->first, key)) - { - return it; - } - } - return Container::end(); + return find_impl(*this, key); } template::value, int> = 0> const_iterator find(KeyType && key) const // NOLINT(cppcoreguidelines-missing-std-forward) { - for (auto it = this->begin(); it != this->end(); ++it) - { - if (m_compare(it->first, key)) - { - return it; - } - } - return Container::end(); + return find_impl(*this, key); } std::pair insert( value_type&& value ) @@ -368,12 +323,10 @@ template , std::pair insert( const value_type& value ) { - for (auto it = this->begin(); it != this->end(); ++it) + const auto it = find_impl(*this, value.first); + if (it != this->end()) { - if (m_compare(it->first, value.first)) - { - return {it, false}; - } + return {it, false}; } append(value); return {--this->end(), true}; diff --git a/tests/src/unit-class_parser.cpp b/tests/src/unit-class_parser.cpp index 75f3757e8..0ae5e382b 100644 --- a/tests/src/unit-class_parser.cpp +++ b/tests/src/unit-class_parser.cpp @@ -2684,12 +2684,10 @@ TEST_CASE("diagnostic positions: value lifetime, input adapters, and SAX") SECTION("move constructor resets the moved-from value to npos") { - // basic_json(basic_json&&) (json.hpp, around line 1951) copies + // basic_json(basic_json&&) copies // other's start_position/end_position into *this and then resets - // other's to npos (see the cppcheck-suppress[accessForwarded] - // annotation there, which flags this reset as worth a second - // look). Only the top-level moved-from value is affected; its - // (moved-away) children are gone along with it. + // other's to npos. Only the top-level moved-from value is + // affected; its (moved-away) children are gone along with it. const std::string s = R"({"a":1,"b":[1,2,3]})"; json a = json::parse(s); const auto a_start = a.start_pos(); diff --git a/tests/src/unit-modifiers.cpp b/tests/src/unit-modifiers.cpp index ee81120cf..532283f95 100644 --- a/tests/src/unit-modifiers.cpp +++ b/tests/src/unit-modifiers.cpp @@ -630,6 +630,49 @@ TEST_CASE("modifiers") } } + SECTION("rvalue at position moves rather than copies") + { + // regression test: insert(pos, basic_json&&) used to forward to + // insert(pos, const basic_json&) because the named rvalue + // reference parameter is itself an lvalue, so it always + // deep-copied its argument instead of moving it + json j_big = std::string(1000, 'x'); + const auto* const original_buffer = j_big.get_ref().data(); + + auto it = j_array.insert(j_array.begin(), std::move(j_big)); + CHECK(j_array.size() == 5); + CHECK(*it == json(std::string(1000, 'x'))); + CHECK((*it).get_ref().data() == original_buffer); + + // the moved-from value is null, the same as after push_back(&&) + CHECK(j_big.is_null()); // NOLINT(bugprone-use-after-move,hicpp-invalid-access-moved) + } + + SECTION("self-aliasing insertion") + { + SECTION("without reallocation") + { + json j_self = {1, 2, 3, 4}; + j_self.get_ref().reserve(j_self.size() + 1); + + auto it = j_self.insert(j_self.begin(), std::move(j_self[1])); + CHECK(j_self.size() == 5); + CHECK(*it == json(2)); + CHECK(j_self == json({2, 1, nullptr, 3, 4})); + } + + SECTION("with reallocation") + { + json j_self = {1, 2, 3, 4}; + j_self.get_ref().shrink_to_fit(); + + auto it = j_self.insert(j_self.begin(), std::move(j_self[1])); + CHECK(j_self.size() == 5); + CHECK(*it == json(2)); + CHECK(j_self == json({2, 1, nullptr, 3, 4})); + } + } + SECTION("copies at position") { SECTION("insert before begin()") From c5650eaa3c91c3ff145e055c4937ee7411cc03d2 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 4 Oct 2026 11:59:03 +0200 Subject: [PATCH 16/22] Reject malformed UTF-16/UTF-32 units in wide-string input (#5704) The wide-string input adapter (used for std::u16string, std::u32string, std::wstring, and iterators over 2- or 4-byte character types) passed some malformed code units on to the lexer as values that are neither a byte (0x00..0xFF) nor char_traits::eof(). As a result: - A lone UTF-16 surrogate inside true/false/null was accepted if its low byte matched the expected letter, or ended the input silently if it was the last unit. - A high surrogate followed by a unit that is not its low surrogate swallowed that unit; if the swallowed unit was the newline ending a // comment, the comment silently extended over the next line. - Where wint_t is a signed int (macOS, the BSDs), a negative wchar_t collided with char_traits::eof() (ending the input early) or was truncated to its low byte, depending on its value. The UTF-32 helper now converts the code unit to std::uint32_t before the range checks, so a negative unit reaches the same "emit 0xFF" branch already used for code points above U+10FFFF. The UTF-16 helper now peeks at the next unit before consuming it, and emits 0xFF instead of the raw surrogate when no valid pair is found, matching how ill-formed UTF-8 bytes are rejected elsewhere in the lexer. Fixes #5645. Signed-off-by: Niels Lohmann --- .../nlohmann/detail/input/input_adapters.hpp | 13 +- single_include/nlohmann/json.hpp | 843 +++++++----------- tests/src/unit-wstring.cpp | 60 +- 3 files changed, 393 insertions(+), 523 deletions(-) diff --git a/include/nlohmann/detail/input/input_adapters.hpp b/include/nlohmann/detail/input/input_adapters.hpp index 7f2f79cbf..9578e275a 100644 --- a/include/nlohmann/detail/input/input_adapters.hpp +++ b/include/nlohmann/detail/input/input_adapters.hpp @@ -453,8 +453,10 @@ struct wide_string_input_helper } else { - // get the current character - const auto wc = input.get_character(); + // get the current character; converted to an unsigned type so that + // a negative unit (wint_t is signed on some platforms) is not + // mistaken for an ASCII character or for EOF + const auto wc = static_cast(input.get_character()); if (wc <= 0x10FFFF) { @@ -522,9 +524,11 @@ struct wide_string_input_helper bool valid_pair = false; if (wc <= 0xDBFF && JSON_HEDLEY_UNLIKELY(!input.empty())) { - const auto wc2 = static_cast(input.get_character()); + // only consume the next unit if it completes the pair + const auto wc2 = static_cast(*input.current); if (0xDC00 <= wc2 && wc2 <= 0xDFFF) { + input.get_character(); const auto charcode = 0x10000u + (((static_cast(wc) & 0x3FFu) << 10u) | (wc2 & 0x3FFu)); utf8_bytes_filled = 0; encode_utf8(charcode, [&utf8_bytes, &utf8_bytes_filled](std::uint32_t byte) @@ -537,7 +541,8 @@ struct wide_string_input_helper if (!valid_pair) { - utf8_bytes[0] = static_cast::int_type>(wc); + // emit a byte that is never valid UTF-8 (see the UTF-32 case) + utf8_bytes[0] = 0xFF; utf8_bytes_filled = 1; } } diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index f8fc9fdb9..3b74fd67f 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -5046,7 +5046,6 @@ inline std::size_t concat_length(const char /*c*/, const Args& ... rest) template inline std::size_t concat_length(const char* cstr, const Args& ... rest) { - // cppcheck-suppress ignoredReturnValue return ::strlen(cstr) + concat_length(rest...); } @@ -8025,8 +8024,10 @@ struct wide_string_input_helper } else { - // get the current character - const auto wc = input.get_character(); + // get the current character; converted to an unsigned type so that + // a negative unit (wint_t is signed on some platforms) is not + // mistaken for an ASCII character or for EOF + const auto wc = static_cast(input.get_character()); if (wc <= 0x10FFFF) { @@ -8094,9 +8095,11 @@ struct wide_string_input_helper bool valid_pair = false; if (wc <= 0xDBFF && JSON_HEDLEY_UNLIKELY(!input.empty())) { - const auto wc2 = static_cast(input.get_character()); + // only consume the next unit if it completes the pair + const auto wc2 = static_cast(*input.current); if (0xDC00 <= wc2 && wc2 <= 0xDFFF) { + input.get_character(); const auto charcode = 0x10000u + (((static_cast(wc) & 0x3FFu) << 10u) | (wc2 & 0x3FFu)); utf8_bytes_filled = 0; encode_utf8(charcode, [&utf8_bytes, &utf8_bytes_filled](std::uint32_t byte) @@ -8109,7 +8112,8 @@ struct wide_string_input_helper if (!valid_pair) { - utf8_bytes[0] = static_cast::int_type>(wc); + // emit a byte that is never valid UTF-8 (see the UTF-32 case) + utf8_bytes[0] = 0xFF; utf8_bytes_filled = 1; } } @@ -8318,7 +8322,7 @@ struct container_input_adapter_factory< ContainerType, { // container is forwarded twice on purpose: the resulting begin/end // iterator types must match adapter_type, computed the same way - // NOLINTNEXTLINE(bugprone-use-after-move) + // NOLINTNEXTLINE(bugprone-use-after-move,hicpp-invalid-access-moved) return input_adapter(begin(std::forward(container)), end(std::forward(container))); } }; @@ -26273,16 +26277,43 @@ template , return *this; } +private: + /// @brief find the entry for @a key, for either constness of @a self + /// @note the single place that performs the linear key search + template + static auto find_impl(Self& self, KeyType&& key) -> decltype(self.begin()) + { + for (auto it = self.begin(); it != self.end(); ++it) + { + if (self.m_compare(it->first, key)) + { + return it; + } + } + return self.end(); + } + + /// @brief remove the entry @a it points to, preserving order + /// @note keys are not movable, so the tail is destroyed and re-constructed in place + void erase_at(iterator it) + { + for (auto next = it; ++next != this->end(); ++it) + { + it->~value_type(); // Destroy but keep allocation + new (&*it) value_type{std::move(*next)}; + } + Container::pop_back(); + } + +public: template::value, int> = 0> std::pair emplace(const key_type& key, V && t) { - for (auto it = this->begin(); it != this->end(); ++it) + const auto it = find_impl(*this, key); + if (it != this->end()) { - if (m_compare(it->first, key)) - { - return {it, false}; - } + return {it, false}; } append(key, std::forward(t)); return {std::prev(this->end()), true}; @@ -26293,12 +26324,10 @@ template , detail::is_constructible>::value, int> = 0> std::pair emplace(KeyType && key, V && t) { - for (auto it = this->begin(); it != this->end(); ++it) + const auto it = find_impl(*this, key); + if (it != this->end()) { - if (m_compare(it->first, key)) - { - return {it, false}; - } + return {it, false}; } append(std::forward(key), std::forward(t)); return {std::prev(this->end()), true}; @@ -26330,75 +26359,55 @@ template , T& at(const key_type& key) { - for (auto it = this->begin(); it != this->end(); ++it) + const auto it = find_impl(*this, key); + if (it == this->end()) { - if (m_compare(it->first, key)) - { - return it->second; - } + JSON_THROW(std::out_of_range("key not found")); } - - JSON_THROW(std::out_of_range("key not found")); + return it->second; } template::value, int> = 0> T & at(KeyType && key) // NOLINT(cppcoreguidelines-missing-std-forward) { - for (auto it = this->begin(); it != this->end(); ++it) + const auto it = find_impl(*this, key); + if (it == this->end()) { - if (m_compare(it->first, key)) - { - return it->second; - } + JSON_THROW(std::out_of_range("key not found")); } - - JSON_THROW(std::out_of_range("key not found")); + return it->second; } const T& at(const key_type& key) const { - for (auto it = this->begin(); it != this->end(); ++it) + const auto it = find_impl(*this, key); + if (it == this->end()) { - if (m_compare(it->first, key)) - { - return it->second; - } + JSON_THROW(std::out_of_range("key not found")); } - - JSON_THROW(std::out_of_range("key not found")); + return it->second; } template::value, int> = 0> const T & at(KeyType && key) const // NOLINT(cppcoreguidelines-missing-std-forward) { - for (auto it = this->begin(); it != this->end(); ++it) + const auto it = find_impl(*this, key); + if (it == this->end()) { - if (m_compare(it->first, key)) - { - return it->second; - } + JSON_THROW(std::out_of_range("key not found")); } - - JSON_THROW(std::out_of_range("key not found")); + return it->second; } size_type erase(const key_type& key) { - for (auto it = this->begin(); it != this->end(); ++it) + const auto it = find_impl(*this, key); + if (it != this->end()) { - if (m_compare(it->first, key)) - { - // Since we cannot move const Keys, re-construct them in place - for (auto next = it; ++next != this->end(); ++it) - { - it->~value_type(); // Destroy but keep allocation - new (&*it) value_type{std::move(*next)}; - } - Container::pop_back(); - return 1; - } + erase_at(it); + return 1; } return 0; } @@ -26407,19 +26416,11 @@ template , detail::is_usable_as_key_type::value, int> = 0> size_type erase(KeyType && key) // NOLINT(cppcoreguidelines-missing-std-forward) { - for (auto it = this->begin(); it != this->end(); ++it) + const auto it = find_impl(*this, key); + if (it != this->end()) { - if (m_compare(it->first, key)) - { - // Since we cannot move const Keys, re-construct them in place - for (auto next = it; ++next != this->end(); ++it) - { - it->~value_type(); // Destroy but keep allocation - new (&*it) value_type{std::move(*next)}; - } - Container::pop_back(); - return 1; - } + erase_at(it); + return 1; } return 0; } @@ -26484,80 +26485,38 @@ template , size_type count(const key_type& key) const { - for (auto it = this->begin(); it != this->end(); ++it) - { - if (m_compare(it->first, key)) - { - return 1; - } - } - return 0; + return find_impl(*this, key) != this->end() ? 1 : 0; } template::value, int> = 0> size_type count(KeyType && key) const // NOLINT(cppcoreguidelines-missing-std-forward) { - for (auto it = this->begin(); it != this->end(); ++it) - { - if (m_compare(it->first, key)) - { - return 1; - } - } - return 0; + return find_impl(*this, key) != this->end() ? 1 : 0; } iterator find(const key_type& key) { - for (auto it = this->begin(); it != this->end(); ++it) - { - if (m_compare(it->first, key)) - { - return it; - } - } - return Container::end(); + return find_impl(*this, key); } template::value, int> = 0> iterator find(KeyType && key) // NOLINT(cppcoreguidelines-missing-std-forward) { - for (auto it = this->begin(); it != this->end(); ++it) - { - if (m_compare(it->first, key)) - { - return it; - } - } - return Container::end(); + return find_impl(*this, key); } const_iterator find(const key_type& key) const { - for (auto it = this->begin(); it != this->end(); ++it) - { - if (m_compare(it->first, key)) - { - return it; - } - } - return Container::end(); + return find_impl(*this, key); } template::value, int> = 0> const_iterator find(KeyType && key) const // NOLINT(cppcoreguidelines-missing-std-forward) { - for (auto it = this->begin(); it != this->end(); ++it) - { - if (m_compare(it->first, key)) - { - return it; - } - } - return Container::end(); + return find_impl(*this, key); } std::pair insert( value_type&& value ) @@ -26567,12 +26526,10 @@ template , std::pair insert( const value_type& value ) { - for (auto it = this->begin(); it != this->end(); ++it) + const auto it = find_impl(*this, value.first); + if (it != this->end()) { - if (m_compare(it->first, value.first)) - { - return {it, false}; - } + return {it, false}; } append(value); return {--this->end(), true}; @@ -26695,11 +26652,12 @@ struct is_std_optional> : std::true_type {}; @brief a class to store JSON values @internal -@invariant The member variables @a m_value and @a m_type have the following -relationship: -- If `m_type == value_t::object`, then `m_value.object != nullptr`. -- If `m_type == value_t::array`, then `m_value.array != nullptr`. -- If `m_type == value_t::string`, then `m_value.string != nullptr`. +@invariant The member variables @a m_data.m_value and @a m_data.m_type have +the following relationship: +- If `m_data.m_type == value_t::object`, then `m_data.m_value.object != nullptr`. +- If `m_data.m_type == value_t::array`, then `m_data.m_value.array != nullptr`. +- If `m_data.m_type == value_t::string`, then `m_data.m_value.string != nullptr`. +- If `m_data.m_type == value_t::binary`, then `m_data.m_value.binary != nullptr`. The invariants are checked by member function assert_invariant(). @note ObjectType trick from https://stackoverflow.com/a/9860911 @@ -26762,18 +26720,12 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec } private: - using primitive_iterator_t = ::nlohmann::detail::primitive_iterator_t; - template - using internal_iterator = ::nlohmann::detail::internal_iterator; template using iter_impl = ::nlohmann::detail::iter_impl; template using iteration_proxy = ::nlohmann::detail::iteration_proxy; template using json_reverse_iterator = ::nlohmann::detail::json_reverse_iterator; - template - using output_adapter_t = ::nlohmann::detail::output_adapter_t; - template using binary_reader = ::nlohmann::detail::binary_reader; template using binary_writer = ::nlohmann::detail::binary_writer; @@ -26914,8 +26866,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec std::to_string(__GNUC_PATCHLEVEL__)) } }; -#elif defined(__HP_cc) || defined(__HP_aCC) - result["compiler"] = "hp" +#elif defined(__HP_aCC) + result["compiler"] = {{"family", "hp"}, {"version", __HP_aCC}}; #elif defined(__IBMCPP__) result["compiler"] = {{"family", "ilecpp"}, {"version", __IBMCPP__}}; #elif defined(_MSC_VER) @@ -27057,9 +27009,9 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec binary | binary | pointer to @ref binary_t null | null | *no value is stored* - @note Variable-length types (objects, arrays, and strings) are stored as - pointers. The size of the union should not exceed 64 bits if the default - value types are used. + @note Variable-length types (objects, arrays, strings, and binary + values) are stored as pointers. The size of the union should not exceed + 64 bits if the default value types are used. @since version 1.0.0 */ @@ -27311,7 +27263,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec end of every constructor to make sure that created objects respect the invariant. Furthermore, it has to be called each time the type of a JSON value is changed, because the invariant expresses a relationship between - @a m_type and @a m_value. + @a m_data.m_type and @a m_data.m_value. Furthermore, the parent relation is checked for arrays and objects: If @a check_parents true and the value is an array or object, then the @@ -27332,7 +27284,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec #if JSON_DIAGNOSTICS JSON_TRY { - // cppcheck-suppress assertWithSideEffect JSON_ASSERT(!check_parents || !is_structured() || std::all_of(begin(), end(), [this](const basic_json & j) { return j.m_parent == this; @@ -28504,8 +28455,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief create a null object /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ - basic_json(std::nullptr_t = nullptr) noexcept // NOLINT(bugprone-exception-escape) - : basic_json(value_t::null) + basic_json(std::nullptr_t = nullptr) noexcept { assert_invariant(); } @@ -28869,15 +28819,15 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief move constructor /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ basic_json(basic_json&& other) noexcept - : json_base_class_t(std::forward(other)), - m_data(std::move(other.m_data)) // cppcheck-suppress[accessForwarded] TODO check + : json_base_class_t(std::move(static_cast(other))), + m_data(std::move(other.m_data)) #if JSON_DIAGNOSTIC_POSITIONS - , start_position(other.start_position) // cppcheck-suppress[accessForwarded] TODO check - , end_position(other.end_position) // cppcheck-suppress[accessForwarded] TODO check + , start_position(other.start_position) + , end_position(other.end_position) #endif { // check that the passed value is valid - other.assert_invariant(false); // cppcheck-suppress[accessForwarded] + other.assert_invariant(false); // invalidate payload other.m_data.m_type = value_t::null; @@ -29444,7 +29394,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec @tparam PointerType pointer type; must be a pointer to @ref array_t, @ref object_t, @ref string_t, @ref boolean_t, @ref number_integer_t, - @ref number_unsigned_t, or @ref number_float_t. + @ref number_unsigned_t, @ref number_float_t, or @ref binary_t. @return pointer to the internally stored JSON value if the requested pointer type @a PointerType fits to the JSON value; `nullptr` otherwise @@ -29612,6 +29562,90 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @} + private: + /// @brief look up @a key in the object held by @a j, for either constness of @a j + /// @note the single place that performs a (possibly transparent) object key lookup + template + static auto object_lookup(Self& j, KeyType&& key) + -> decltype(j.m_data.m_value.object->find(lookup_key(std::forward(key)))) + { + return j.m_data.m_value.object->find(lookup_key(std::forward(key))); + } + + /// @brief checked object element access used by the at() overloads taking a key + /// @throw type_error.304 if @a j is not an object + /// @throw out_of_range.403 if @a key is not found + template + static auto object_at(Self& j, KeyType&& key) + -> decltype((object_lookup(j, std::forward(key))->second)) + { + // at only works for objects + if (JSON_HEDLEY_UNLIKELY(!j.is_object())) + { + JSON_THROW(type_error::create(304, detail::concat("cannot use at() with ", j.type_name()), &j)); + } + + auto it = object_lookup(j, std::forward(key)); + if (it == j.m_data.m_value.object->end()) + { + // key is only forwarded into the lookup above: object_t::find() (a plain + // std::map or ordered_map) never moves from its argument, so key is still + // valid here regardless of whether KeyType was deduced as an rvalue reference + // NOLINTNEXTLINE(bugprone-use-after-move,hicpp-invalid-access-moved) + JSON_THROW(out_of_range::create(403, detail::concat("key '", string_t(key), "' not found"), &j)); + } + return it->second; + } + + /// @brief checked array element access used by the at() overloads taking an index + /// @throw type_error.304 if @a j is not an array + /// @throw out_of_range.401 if @a idx is out of range + template + static auto array_at(Self& j, size_type idx) + -> decltype((*j.m_data.m_value.array)[idx]) + { + // at only works for arrays + if (JSON_HEDLEY_UNLIKELY(!j.is_array())) + { + JSON_THROW(type_error::create(304, detail::concat("cannot use at() with ", j.type_name()), &j)); + } + + if (JSON_HEDLEY_UNLIKELY(idx >= j.m_data.m_value.array->size())) + { + JSON_THROW(out_of_range::create(401, detail::concat("array index ", std::to_string(idx), " is out of range"), &j)); + } + + return (*j.m_data.m_value.array)[idx]; + } + + /// @brief convert a null value to an empty container of type @a Container + /// @tparam Container array_t or object_t; any other type does not compile + template + void convert_null_to() + { + JSON_ASSERT(is_null()); + // create the container before touching the type, so a throwing + // allocation leaves this value as a valid null rather than a type + // tag with a dangling/null pointer behind it + set_container(create()); + assert_invariant(); + } + + /// @brief store a freshly created array and set the matching type + void set_container(array_t* array) noexcept + { + m_data.m_value.array = array; + m_data.m_type = value_t::array; + } + + /// @brief store a freshly created object and set the matching type + void set_container(object_t* object) noexcept + { + m_data.m_value.object = object; + m_data.m_type = value_t::object; + } + + public: //////////////////// // element access // //////////////////// @@ -29624,54 +29658,21 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/at/ reference at(size_type idx) { - // at only works for arrays - if (JSON_HEDLEY_UNLIKELY(!is_array())) - { - JSON_THROW(type_error::create(304, detail::concat("cannot use at() with ", type_name()), this)); - } - - if (JSON_HEDLEY_UNLIKELY(idx >= m_data.m_value.array->size())) - { - JSON_THROW(out_of_range::create(401, detail::concat("array index ", std::to_string(idx), " is out of range"), this)); - } - - return set_parent((*m_data.m_value.array)[idx]); + return set_parent(array_at(*this, idx)); } /// @brief access specified array element with bounds checking /// @sa https://json.nlohmann.me/api/basic_json/at/ const_reference at(size_type idx) const { - // at only works for arrays - if (JSON_HEDLEY_UNLIKELY(!is_array())) - { - JSON_THROW(type_error::create(304, detail::concat("cannot use at() with ", type_name()), this)); - } - - if (JSON_HEDLEY_UNLIKELY(idx >= m_data.m_value.array->size())) - { - JSON_THROW(out_of_range::create(401, detail::concat("array index ", std::to_string(idx), " is out of range"), this)); - } - - return (*m_data.m_value.array)[idx]; + return array_at(*this, idx); } /// @brief access specified object element with bounds checking /// @sa https://json.nlohmann.me/api/basic_json/at/ reference at(const typename object_t::key_type& key) { - // at only works for objects - if (JSON_HEDLEY_UNLIKELY(!is_object())) - { - JSON_THROW(type_error::create(304, detail::concat("cannot use at() with ", type_name()), this)); - } - - auto it = m_data.m_value.object->find(key); - if (it == m_data.m_value.object->end()) - { - JSON_THROW(out_of_range::create(403, detail::concat("key '", key, "' not found"), this)); - } - return set_parent(it->second); + return set_parent(object_at(*this, key)); } /// @brief access specified object element with bounds checking @@ -29680,36 +29681,14 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec detail::is_usable_as_basic_json_key_type::value, int> = 0> reference at(KeyType && key) { - // at only works for objects - if (JSON_HEDLEY_UNLIKELY(!is_object())) - { - JSON_THROW(type_error::create(304, detail::concat("cannot use at() with ", type_name()), this)); - } - - auto it = m_data.m_value.object->find(lookup_key(std::forward(key))); - if (it == m_data.m_value.object->end()) - { - JSON_THROW(out_of_range::create(403, detail::concat("key '", string_t(std::forward(key)), "' not found"), this)); - } - return set_parent(it->second); + return set_parent(object_at(*this, std::forward(key))); } /// @brief access specified object element with bounds checking /// @sa https://json.nlohmann.me/api/basic_json/at/ const_reference at(const typename object_t::key_type& key) const { - // at only works for objects - if (JSON_HEDLEY_UNLIKELY(!is_object())) - { - JSON_THROW(type_error::create(304, detail::concat("cannot use at() with ", type_name()), this)); - } - - auto it = m_data.m_value.object->find(key); - if (it == m_data.m_value.object->end()) - { - JSON_THROW(out_of_range::create(403, detail::concat("key '", key, "' not found"), this)); - } - return it->second; + return object_at(*this, key); } /// @brief access specified object element with bounds checking @@ -29718,18 +29697,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec detail::is_usable_as_basic_json_key_type::value, int> = 0> const_reference at(KeyType && key) const { - // at only works for objects - if (JSON_HEDLEY_UNLIKELY(!is_object())) - { - JSON_THROW(type_error::create(304, detail::concat("cannot use at() with ", type_name()), this)); - } - - auto it = m_data.m_value.object->find(lookup_key(std::forward(key))); - if (it == m_data.m_value.object->end()) - { - JSON_THROW(out_of_range::create(403, detail::concat("key '", string_t(std::forward(key)), "' not found"), this)); - } - return it->second; + return object_at(*this, std::forward(key)); } /// @brief access specified array element @@ -29739,9 +29707,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // implicitly convert a null value to an empty array if (is_null()) { - m_data.m_type = value_t::array; - m_data.m_value.array = create(); - assert_invariant(); + convert_null_to(); } // operator[] only works for arrays @@ -29807,9 +29773,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // implicitly convert a null value to an empty object if (is_null()) { - m_data.m_type = value_t::object; - m_data.m_value.object = create(); - assert_invariant(); + convert_null_to(); } // operator[] only works for objects @@ -29829,7 +29793,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // const operator[] only works for objects if (JSON_HEDLEY_LIKELY(is_object())) { - auto it = m_data.m_value.object->find(key); + auto it = object_lookup(*this, key); JSON_ASSERT(it != m_data.m_value.object->end()); return it->second; } @@ -29860,9 +29824,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // implicitly convert a null value to an empty object if (is_null()) { - m_data.m_type = value_t::object; - m_data.m_value.object = create(); - assert_invariant(); + convert_null_to(); } // operator[] only works for objects @@ -29884,7 +29846,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // const operator[] only works for objects if (JSON_HEDLEY_LIKELY(is_object())) { - auto it = m_data.m_value.object->find(lookup_key(std::forward(key))); + auto it = object_lookup(*this, std::forward(key)); JSON_ASSERT(it != m_data.m_value.object->end()); return it->second; } @@ -29904,6 +29866,36 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec detail::is_c_string_uncvref::value, string_t, typename std::decay::type >; + /// @brief look up @a key for value(), the single place shared by all key-based overloads + /// @throw type_error.306 if this is not an object + /// @return a pointer to the found value, or `nullptr` if @a key was not found + template + const basic_json* value_member(KeyType&& key) const + { + // value only works for objects + if (JSON_HEDLEY_UNLIKELY(!is_object())) + { + JSON_THROW(type_error::create(306, detail::concat("cannot use value() with ", type_name()), this)); + } + + const auto it = find(std::forward(key)); + return it != end() ? &*it : nullptr; + } + + /// @brief resolve @a ptr for value(), the single place shared by both json_pointer overloads + /// @throw type_error.306 if this is not an array or object + /// @return a pointer to the resolved value, or `nullptr` if @a ptr does not resolve + const basic_json* value_pointee(const json_pointer& ptr) const + { + // value only works for arrays and objects + if (JSON_HEDLEY_UNLIKELY(!is_structured())) + { + JSON_THROW(type_error::create(306, detail::concat("cannot use value() with ", type_name()), this)); + } + + return ptr.get_checked_or_null(this); + } + public: // an integer literal 0 would otherwise convert to a null const char* and from there to key_type template::value, int> = 0> @@ -29917,20 +29909,9 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec && !std::is_same>::value, int > = 0 > ValueType value(const typename object_t::key_type& key, const ValueType& default_value) const { - // value only works for objects - if (JSON_HEDLEY_LIKELY(is_object())) - { - // If 'key' is found, return its value. Otherwise, return `default_value'. - const auto it = find(key); - if (it != end()) - { - return it->template get(); - } - - return default_value; - } - - JSON_THROW(type_error::create(306, detail::concat("cannot use value() with ", type_name()), this)); + // If 'key' is found, return its value. Otherwise, return `default_value'. + const auto* found = value_member(key); + return found != nullptr ? found->template get() : default_value; } /// @brief access specified object element with default value @@ -29942,20 +29923,9 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec && !std::is_same>::value, int > = 0 > ReturnType value(const typename object_t::key_type& key, ValueType && default_value) const { - // value only works for objects - if (JSON_HEDLEY_LIKELY(is_object())) - { - // If 'key' is found, return its value. Otherwise, return `default_value'. - const auto it = find(key); - if (it != end()) - { - return it->template get(); - } - - return std::forward(default_value); - } - - JSON_THROW(type_error::create(306, detail::concat("cannot use value() with ", type_name()), this)); + // If 'key' is found, return its value. Otherwise, return `default_value'. + const auto* found = value_member(key); + return found != nullptr ? found->template get() : std::forward(default_value); } /// @brief access specified object element with default value @@ -29968,23 +29938,12 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec && !std::is_same>::value, int > = 0 > ValueType value(KeyType && key, const ValueType& default_value) const { - // value only works for objects - if (JSON_HEDLEY_LIKELY(is_object())) - { - // If 'key' is found, return its value. Otherwise, return `default_value'. - const auto it = find(std::forward(key)); - if (it != end()) - { - return it->template get(); - } - - return default_value; - } - - JSON_THROW(type_error::create(306, detail::concat("cannot use value() with ", type_name()), this)); + // If 'key' is found, return its value. Otherwise, return `default_value'. + const auto* found = value_member(std::forward(key)); + return found != nullptr ? found->template get() : default_value; } - /// @brief access specified object element via JSON Pointer with default value + /// @brief access specified object element with default value /// @sa https://json.nlohmann.me/api/basic_json/value/ template < class ValueType, class KeyType, class ReturnType = typename value_return_type::type, detail::enable_if_t < @@ -29995,20 +29954,9 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec && !std::is_same>::value, int > = 0 > ReturnType value(KeyType && key, ValueType && default_value) const { - // value only works for objects - if (JSON_HEDLEY_LIKELY(is_object())) - { - // If 'key' is found, return its value. Otherwise, return `default_value'. - const auto it = find(std::forward(key)); - if (it != end()) - { - return it->template get(); - } - - return std::forward(default_value); - } - - JSON_THROW(type_error::create(306, detail::concat("cannot use value() with ", type_name()), this)); + // If 'key' is found, return its value. Otherwise, return `default_value'. + const auto* found = value_member(std::forward(key)); + return found != nullptr ? found->template get() : std::forward(default_value); } /// @brief access specified object element via JSON Pointer with default value @@ -30018,21 +29966,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec && !std::is_same>::value, int > = 0 > ValueType value(const json_pointer& ptr, const ValueType& default_value) const { - // value only works for arrays and objects - if (JSON_HEDLEY_LIKELY(is_structured())) - { - // If the pointer resolves to a value, return it. Otherwise, return - // 'default_value'. - const auto* res = ptr.get_checked_or_null(this); - if (JSON_HEDLEY_LIKELY(res != nullptr)) - { - return res->template get(); - } - - return default_value; - } - - JSON_THROW(type_error::create(306, detail::concat("cannot use value() with ", type_name()), this)); + // If the pointer resolves to a value, return it. Otherwise, return + // 'default_value'. + const auto* found = value_pointee(ptr); + return found != nullptr ? found->template get() : default_value; } /// @brief access specified object element via JSON Pointer with default value @@ -30043,21 +29980,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec && !std::is_same>::value, int > = 0 > ReturnType value(const json_pointer& ptr, ValueType && default_value) const { - // value only works for arrays and objects - if (JSON_HEDLEY_LIKELY(is_structured())) - { - // If the pointer resolves to a value, return it. Otherwise, return - // 'default_value'. - const auto* res = ptr.get_checked_or_null(this); - if (JSON_HEDLEY_LIKELY(res != nullptr)) - { - return res->template get(); - } - - return std::forward(default_value); - } - - JSON_THROW(type_error::create(306, detail::concat("cannot use value() with ", type_name()), this)); + // If the pointer resolves to a value, return it. Otherwise, return + // 'default_value'. + const auto* found = value_pointee(ptr); + return found != nullptr ? found->template get() : std::forward(default_value); } template < class ValueType, class BasicJsonType, detail::enable_if_t < @@ -30142,21 +30068,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec JSON_THROW(invalid_iterator::create(205, "iterator out of range", this)); } - if (is_string()) - { - AllocatorType alloc; - std::allocator_traits::destroy(alloc, m_data.m_value.string); - std::allocator_traits::deallocate(alloc, m_data.m_value.string, 1); - m_data.m_value.string = nullptr; - } - else if (is_binary()) - { - AllocatorType alloc; - std::allocator_traits::destroy(alloc, m_data.m_value.binary); - std::allocator_traits::deallocate(alloc, m_data.m_value.binary, 1); - m_data.m_value.binary = nullptr; - } - + m_data.m_value.destroy(m_data.m_type); + m_data.m_value = {}; m_data.m_type = value_t::null; assert_invariant(); break; @@ -30208,27 +30121,14 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec case value_t::string: case value_t::binary: { - if (JSON_HEDLEY_LIKELY(!first.m_it.primitive_iterator.is_begin() - || !last.m_it.primitive_iterator.is_end())) + if (JSON_HEDLEY_UNLIKELY(!first.m_it.primitive_iterator.is_begin() + || !last.m_it.primitive_iterator.is_end())) { JSON_THROW(invalid_iterator::create(204, "iterators out of range", this)); } - if (is_string()) - { - AllocatorType alloc; - std::allocator_traits::destroy(alloc, m_data.m_value.string); - std::allocator_traits::deallocate(alloc, m_data.m_value.string, 1); - m_data.m_value.string = nullptr; - } - else if (is_binary()) - { - AllocatorType alloc; - std::allocator_traits::destroy(alloc, m_data.m_value.binary); - std::allocator_traits::deallocate(alloc, m_data.m_value.binary, 1); - m_data.m_value.binary = nullptr; - } - + m_data.m_value.destroy(m_data.m_type); + m_data.m_value = {}; m_data.m_type = value_t::null; assert_invariant(); break; @@ -30284,7 +30184,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec JSON_THROW(type_error::create(307, detail::concat("cannot use erase() with ", type_name()), this)); } - const auto it = m_data.m_value.object->find(lookup_key(std::forward(key))); + const auto it = object_lookup(*this, std::forward(key)); if (it != m_data.m_value.object->end()) { m_data.m_value.object->erase(it); @@ -30361,7 +30261,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec if (is_object()) { - result.m_it.object_iterator = m_data.m_value.object->find(key); + result.m_it.object_iterator = object_lookup(*this, key); } return result; @@ -30375,7 +30275,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec if (is_object()) { - result.m_it.object_iterator = m_data.m_value.object->find(key); + result.m_it.object_iterator = object_lookup(*this, key); } return result; @@ -30391,7 +30291,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec if (is_object()) { - result.m_it.object_iterator = m_data.m_value.object->find(lookup_key(std::forward(key))); + result.m_it.object_iterator = object_lookup(*this, std::forward(key)); } return result; @@ -30407,7 +30307,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec if (is_object()) { - result.m_it.object_iterator = m_data.m_value.object->find(lookup_key(std::forward(key))); + result.m_it.object_iterator = object_lookup(*this, std::forward(key)); } return result; @@ -30438,7 +30338,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec JSON_HEDLEY_WARN_UNUSED_RESULT bool contains(const typename object_t::key_type& key) const { - return is_object() && m_data.m_value.object->find(key) != m_data.m_value.object->end(); + return is_object() && object_lookup(*this, key) != m_data.m_value.object->end(); } /// @brief check the existence of an element in a JSON object @@ -30448,7 +30348,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec JSON_HEDLEY_WARN_UNUSED_RESULT bool contains(KeyType && key) const { - return is_object() && m_data.m_value.object->find(lookup_key(std::forward(key))) != m_data.m_value.object->end(); + return is_object() && object_lookup(*this, std::forward(key)) != m_data.m_value.object->end(); } /// @brief check the existence of an element in a JSON object given a JSON pointer @@ -30813,9 +30713,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // transform a null object into an array if (is_null()) { - m_data.m_type = value_t::array; - m_data.m_value = value_t::array; - assert_invariant(); + convert_null_to(); } // add the element to the array (move semantics) @@ -30846,9 +30744,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // transform a null object into an array if (is_null()) { - m_data.m_type = value_t::array; - m_data.m_value = value_t::array; - assert_invariant(); + convert_null_to(); } // add the element to the array @@ -30878,9 +30774,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // transform a null object into an object if (is_null()) { - m_data.m_type = value_t::object; - m_data.m_value = value_t::object; - assert_invariant(); + convert_null_to(); } // add the element to the object @@ -30934,9 +30828,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // transform a null object into an array if (is_null()) { - m_data.m_type = value_t::array; - m_data.m_value = value_t::array; - assert_invariant(); + convert_null_to(); } // add the element to the array (perfect forwarding) @@ -30959,9 +30851,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // transform a null object into an object if (is_null()) { - m_data.m_type = value_t::object; - m_data.m_value = value_t::object; - assert_invariant(); + convert_null_to(); } // add the element to the array (perfect forwarding) @@ -31021,7 +30911,22 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/insert/ iterator insert(const_iterator pos, basic_json&& val) // NOLINT(performance-unnecessary-value-param) { - return insert(std::move(pos), val); + // insert only works for arrays + if (JSON_HEDLEY_LIKELY(is_array())) + { + // check if iterator pos fits to this JSON value + if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) + { + JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value", this)); + } + + // moving into a local first keeps this safe even if val aliases + // an element of this array + basic_json tmp(std::move(val)); + return insert_iterator(pos, std::move(tmp)); + } + + JSON_THROW(type_error::create(309, detail::concat("cannot use insert() with ", type_name()), this)); } /// @brief inserts copies of element into array @@ -31197,14 +31102,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// value is an object; called first by both @ref update overloads void prepare_update() { - // implicitly convert a null value to an empty object; create the - // object before setting the type, so a throwing allocation leaves - // this value null + // implicitly convert a null value to an empty object if (is_null()) { - m_data.m_value.object = create(); - m_data.m_type = value_t::object; - assert_invariant(); + convert_null_to(); } if (JSON_HEDLEY_UNLIKELY(!is_object())) @@ -31413,7 +31314,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/swap/ void swap(binary_t& other) // NOLINT(bugprone-exception-escape,cppcoreguidelines-noexcept-swap,performance-noexcept-swap) { - // swap only works for strings + // swap only works for binary values if (JSON_HEDLEY_LIKELY(is_binary())) { using std::swap; @@ -31429,7 +31330,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/swap/ void swap(typename binary_t::container_type& other) // NOLINT(bugprone-exception-escape) { - // swap only works for strings + // swap only works for binary values if (JSON_HEDLEY_LIKELY(is_binary())) { using std::swap; @@ -31930,7 +31831,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool ignore_trailing_commas = false) { basic_json result; - parser(detail::input_adapter(std::forward(i)), std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas).parse(true, result); // cppcheck-suppress[accessMoved,accessForwarded] + auto p = parser(detail::input_adapter(std::forward(i)), std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas); + p.parse(true, result); return result; } @@ -31947,7 +31849,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool ignore_trailing_commas = false) { basic_json result; - parser(detail::input_adapter(std::move(first), std::move(last)), std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas).parse(true, result); // cppcheck-suppress[accessMoved] + auto p = parser(detail::input_adapter(std::move(first), std::move(last)), std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas); + p.parse(true, result); return result; } @@ -31960,7 +31863,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool ignore_trailing_commas = false) { basic_json result; - parser(i.get(), std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas).parse(true, result); // cppcheck-suppress[accessMoved] + auto p = parser(i.get(), std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas); + p.parse(true, result); return result; } @@ -32187,6 +32091,31 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec } #endif + private: + /*! + @brief shared implementation of the binary from_cbor/from_msgpack/ + from_ubjson/from_bjdata/from_bon8/from_bson overloads + + Building the @ref binary_reader as a named local, rather than as a + temporary that @ref detail::json_sax_dom_parser::parse is called on in + the same expression, avoids a false-positive cppcheck accessMoved + warning in each of the 16 callers. + */ + template + static basic_json from_binary_impl(InputAdapterType ia, const input_format_t format, + const bool strict, const bool allow_exceptions, + const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + binary_reader reader(std::move(ia), format); + if (!reader.sax_parse(&sdp, strict, tag_handler)) + { + result = value_t::discarded; + } + return result; + } + ////////////////////////////////////////// // binary serialization/deserialization // ////////////////////////////////////////// @@ -32359,14 +32288,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool allow_exceptions = true, const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error) { - basic_json result; - auto ia = detail::input_adapter(std::forward(i)); - detail::json_sax_dom_parser sdp(result, allow_exceptions); - if (!binary_reader(std::move(ia), input_format_t::cbor).sax_parse(&sdp, strict, tag_handler)) // cppcheck-suppress[accessMoved] - { - result = value_t::discarded; - } - return result; + return from_binary_impl(detail::input_adapter(std::forward(i)), input_format_t::cbor, strict, allow_exceptions, tag_handler); } /// @brief create a JSON value from an input in CBOR format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -32379,14 +32301,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool allow_exceptions = true, const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error) { - basic_json result; - auto ia = detail::input_adapter(std::move(first), std::move(last)); - detail::json_sax_dom_parser sdp(result, allow_exceptions); - if (!binary_reader(std::move(ia), input_format_t::cbor).sax_parse(&sdp, strict, tag_handler)) // cppcheck-suppress[accessMoved] - { - result = value_t::discarded; - } - return result; + return from_binary_impl(detail::input_adapter(std::move(first), std::move(last)), input_format_t::cbor, strict, allow_exceptions, tag_handler); } template @@ -32407,15 +32322,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool allow_exceptions = true, const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error) { - basic_json result; - auto ia = i.get(); - detail::json_sax_dom_parser sdp(result, allow_exceptions); - // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) - if (!binary_reader(std::move(ia), input_format_t::cbor).sax_parse(&sdp, strict, tag_handler)) // cppcheck-suppress[accessMoved] - { - result = value_t::discarded; - } - return result; + return from_binary_impl(i.get(), input_format_t::cbor, strict, allow_exceptions, tag_handler); } /// @brief create a JSON value from an input in MessagePack format @@ -32426,14 +32333,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool strict = true, const bool allow_exceptions = true) { - basic_json result; - auto ia = detail::input_adapter(std::forward(i)); - detail::json_sax_dom_parser sdp(result, allow_exceptions); - if (!binary_reader(std::move(ia), input_format_t::msgpack).sax_parse(&sdp, strict)) // cppcheck-suppress[accessMoved] - { - result = value_t::discarded; - } - return result; + return from_binary_impl(detail::input_adapter(std::forward(i)), input_format_t::msgpack, strict, allow_exceptions); } /// @brief create a JSON value from an input in MessagePack format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -32445,14 +32345,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool strict = true, const bool allow_exceptions = true) { - basic_json result; - auto ia = detail::input_adapter(std::move(first), std::move(last)); - detail::json_sax_dom_parser sdp(result, allow_exceptions); - if (!binary_reader(std::move(ia), input_format_t::msgpack).sax_parse(&sdp, strict)) // cppcheck-suppress[accessMoved] - { - result = value_t::discarded; - } - return result; + return from_binary_impl(detail::input_adapter(std::move(first), std::move(last)), input_format_t::msgpack, strict, allow_exceptions); } template @@ -32471,15 +32364,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool strict = true, const bool allow_exceptions = true) { - basic_json result; - auto ia = i.get(); - detail::json_sax_dom_parser sdp(result, allow_exceptions); - // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) - if (!binary_reader(std::move(ia), input_format_t::msgpack).sax_parse(&sdp, strict)) // cppcheck-suppress[accessMoved] - { - result = value_t::discarded; - } - return result; + return from_binary_impl(i.get(), input_format_t::msgpack, strict, allow_exceptions); } /// @brief create a JSON value from an input in UBJSON format @@ -32490,14 +32375,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool strict = true, const bool allow_exceptions = true) { - basic_json result; - auto ia = detail::input_adapter(std::forward(i)); - detail::json_sax_dom_parser sdp(result, allow_exceptions); - if (!binary_reader(std::move(ia), input_format_t::ubjson).sax_parse(&sdp, strict)) // cppcheck-suppress[accessMoved] - { - result = value_t::discarded; - } - return result; + return from_binary_impl(detail::input_adapter(std::forward(i)), input_format_t::ubjson, strict, allow_exceptions); } /// @brief create a JSON value from an input in UBJSON format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -32509,14 +32387,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool strict = true, const bool allow_exceptions = true) { - basic_json result; - auto ia = detail::input_adapter(std::move(first), std::move(last)); - detail::json_sax_dom_parser sdp(result, allow_exceptions); - if (!binary_reader(std::move(ia), input_format_t::ubjson).sax_parse(&sdp, strict)) // cppcheck-suppress[accessMoved] - { - result = value_t::discarded; - } - return result; + return from_binary_impl(detail::input_adapter(std::move(first), std::move(last)), input_format_t::ubjson, strict, allow_exceptions); } template @@ -32535,15 +32406,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool strict = true, const bool allow_exceptions = true) { - basic_json result; - auto ia = i.get(); - detail::json_sax_dom_parser sdp(result, allow_exceptions); - // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) - if (!binary_reader(std::move(ia), input_format_t::ubjson).sax_parse(&sdp, strict)) // cppcheck-suppress[accessMoved] - { - result = value_t::discarded; - } - return result; + return from_binary_impl(i.get(), input_format_t::ubjson, strict, allow_exceptions); } /// @brief create a JSON value from an input in BJData format @@ -32554,14 +32417,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool strict = true, const bool allow_exceptions = true) { - basic_json result; - auto ia = detail::input_adapter(std::forward(i)); - detail::json_sax_dom_parser sdp(result, allow_exceptions); - if (!binary_reader(std::move(ia), input_format_t::bjdata).sax_parse(&sdp, strict)) // cppcheck-suppress[accessMoved] - { - result = value_t::discarded; - } - return result; + return from_binary_impl(detail::input_adapter(std::forward(i)), input_format_t::bjdata, strict, allow_exceptions); } /// @brief create a JSON value from an input in BJData format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -32573,14 +32429,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool strict = true, const bool allow_exceptions = true) { - basic_json result; - auto ia = detail::input_adapter(std::move(first), std::move(last)); - detail::json_sax_dom_parser sdp(result, allow_exceptions); - if (!binary_reader(std::move(ia), input_format_t::bjdata).sax_parse(&sdp, strict)) // cppcheck-suppress[accessMoved] - { - result = value_t::discarded; - } - return result; + return from_binary_impl(detail::input_adapter(std::move(first), std::move(last)), input_format_t::bjdata, strict, allow_exceptions); } /// @brief create a JSON value from an input in BON8 format @@ -32591,14 +32440,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool strict = true, const bool allow_exceptions = true) { - basic_json result; - auto ia = detail::input_adapter(std::forward(i)); - detail::json_sax_dom_parser sdp(result, allow_exceptions); - if (!binary_reader(std::move(ia), input_format_t::bon8).sax_parse(&sdp, strict)) // cppcheck-suppress[accessMoved] - { - result = value_t::discarded; - } - return result; + return from_binary_impl(detail::input_adapter(std::forward(i)), input_format_t::bon8, strict, allow_exceptions); } /// @brief create a JSON value from an input in BON8 format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -32610,14 +32452,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool strict = true, const bool allow_exceptions = true) { - basic_json result; - auto ia = detail::input_adapter(std::move(first), std::move(last)); - detail::json_sax_dom_parser sdp(result, allow_exceptions); - if (!binary_reader(std::move(ia), input_format_t::bon8).sax_parse(&sdp, strict)) // cppcheck-suppress[accessMoved] - { - result = value_t::discarded; - } - return result; + return from_binary_impl(detail::input_adapter(std::move(first), std::move(last)), input_format_t::bon8, strict, allow_exceptions); } /// @brief create a JSON value from an input in BSON format @@ -32628,14 +32463,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool strict = true, const bool allow_exceptions = true) { - basic_json result; - auto ia = detail::input_adapter(std::forward(i)); - detail::json_sax_dom_parser sdp(result, allow_exceptions); - if (!binary_reader(std::move(ia), input_format_t::bson).sax_parse(&sdp, strict)) // cppcheck-suppress[accessMoved] - { - result = value_t::discarded; - } - return result; + return from_binary_impl(detail::input_adapter(std::forward(i)), input_format_t::bson, strict, allow_exceptions); } /// @brief create a JSON value from an input in BSON format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -32647,14 +32475,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool strict = true, const bool allow_exceptions = true) { - basic_json result; - auto ia = detail::input_adapter(std::move(first), std::move(last)); - detail::json_sax_dom_parser sdp(result, allow_exceptions); - if (!binary_reader(std::move(ia), input_format_t::bson).sax_parse(&sdp, strict)) // cppcheck-suppress[accessMoved] - { - result = value_t::discarded; - } - return result; + return from_binary_impl(detail::input_adapter(std::move(first), std::move(last)), input_format_t::bson, strict, allow_exceptions); } template @@ -32673,15 +32494,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool strict = true, const bool allow_exceptions = true) { - basic_json result; - auto ia = i.get(); - detail::json_sax_dom_parser sdp(result, allow_exceptions); - // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) - if (!binary_reader(std::move(ia), input_format_t::bson).sax_parse(&sdp, strict)) // cppcheck-suppress[accessMoved] - { - result = value_t::discarded; - } - return result; + return from_binary_impl(i.get(), input_format_t::bson, strict, allow_exceptions); } /// @} diff --git a/tests/src/unit-wstring.cpp b/tests/src/unit-wstring.cpp index b553ce246..c71684002 100644 --- a/tests/src/unit-wstring.cpp +++ b/tests/src/unit-wstring.cpp @@ -8,6 +8,7 @@ #include "doctest_compatibility.h" +#include #include using nlohmann::json; @@ -68,15 +69,15 @@ TEST_CASE("wide strings") CHECK_THROWS_AS(_ = json::parse(w), json::parse_error&); // a lone low surrogate cannot start a pair - CHECK_THROWS_WITH_AS(_ = json::parse(std::u16string{u'"', 0xDC00, u'"'}), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: ill-formed UTF-8 byte; last read: '\"'", json::parse_error&); + CHECK_THROWS_WITH_AS(_ = json::parse(std::u16string{u'"', 0xDC00, u'"'}), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: ill-formed UTF-8 byte; last read: '\"\xFF'", json::parse_error&); // a high surrogate followed by a non-low-surrogate unit is invalid - CHECK_THROWS_WITH_AS(_ = json::parse(std::u16string{u'"', 0xD800, u'a', u'"'}), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: ill-formed UTF-8 byte; last read: '\"'", json::parse_error&); + CHECK_THROWS_WITH_AS(_ = json::parse(std::u16string{u'"', 0xD800, u'a', u'"'}), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: ill-formed UTF-8 byte; last read: '\"\xFF'", json::parse_error&); // ... also when the unit is above the low surrogates - CHECK_THROWS_WITH_AS(_ = json::parse(std::u16string{u'"', 0xD800, 0xE000, u'"'}), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: ill-formed UTF-8 byte; last read: '\"'", json::parse_error&); + CHECK_THROWS_WITH_AS(_ = json::parse(std::u16string{u'"', 0xD800, 0xE000, u'"'}), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: ill-formed UTF-8 byte; last read: '\"\xFF'", json::parse_error&); // a lone low surrogate must not swallow the following unit: pairing // it with any second unit would produce valid UTF-8, so the error // has to report an ill-formed byte at the surrogate's own position - CHECK_THROWS_WITH_AS(_ = json::parse(std::u16string{u'"', 0xDC00, u'a', u'"'}), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: ill-formed UTF-8 byte; last read: '\"'", json::parse_error&); + CHECK_THROWS_WITH_AS(_ = json::parse(std::u16string{u'"', 0xDC00, u'a', u'"'}), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: ill-formed UTF-8 byte; last read: '\"\xFF'", json::parse_error&); // a valid surrogate pair is still decoded (U+1F600) CHECK(json::parse(std::u16string{u'"', 0xD83D, 0xDE00, u'"'}).get() == "\xF0\x9F\x98\x80"); } @@ -104,4 +105,55 @@ TEST_CASE("wide strings") // the same unit inside a string is reported as an ill-formed byte CHECK_THROWS_WITH_AS(_ = json::parse(std::u32string{U'"', static_cast(0xFFFFFFFF), U'"'}), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: ill-formed UTF-8 byte; last read: '\"\xFF'", json::parse_error&); } + + SECTION("malformed wide-string input outside strings (#5645)") + { + json _; + + // a lone low surrogate inside a literal must not be truncated to its + // low byte and mistaken for the letter the literal expects next + // (0xDC72 truncates to 'r', which is what "true" expects after 't') + CHECK_THROWS_WITH_AS(_ = json::parse(std::u16string{u't', static_cast(0xDC72), u'u', u'e'}), + "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid literal; last read: 't\xFF'", json::parse_error&); + + // ... also when the lone surrogate is the last unit of the input + CHECK_THROWS_WITH_AS(_ = json::parse(std::u16string{u'f', u'a', u'l', u's', static_cast(0xDD65)}), + "[json.exception.parse_error.101] parse error at line 1, column 5: syntax error while parsing value - invalid literal; last read: 'fals\xFF'", json::parse_error&); + + // a high surrogate followed by a unit that is not its low surrogate + // must not silently swallow that unit + CHECK_THROWS_WITH_AS(_ = json::parse(std::u16string{u't', static_cast(0xD872), u'X', u'u', u'e'}), + "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid literal; last read: 't\xFF'", json::parse_error&); + + // ... in particular, if the swallowed unit is the newline that ends a + // // comment, the comment must not extend over the following line + CHECK(json::parse(std::u16string{u'[', u'1', u' ', u'/', u'/', static_cast(0xD800), u'\n', + u',', u'2', u' ', u'/', u'/', u'\n', u']'}, + nullptr, true, /*ignore_comments*/true) == json::parse("[1,2]")); + CHECK(json::accept(std::u16string{u'[', u'1', u' ', u'/', u'/', static_cast(0xD800), u'\n', + u',', u'2', u' ', u'/', u'/', u'\n', u']'}, /*ignore_comments*/true)); + + // cases 5 and 6 use a 32-bit wchar_t (Linux, macOS, the BSDs) to reach + // the UTF-32 helper tested above via u32string; the 16-bit wchar_t of + // Windows goes through the UTF-16 helper instead, already covered by + // the u16string cases above +#if WCHAR_MAX > 0xFFFFu + // a negative wchar_t must not be mistaken for + // char_traits::eof() and silently end the input, letting + // trailing garbage pass the strict end-of-input check (only observable + // where wint_t is signed, e.g. macOS/the BSDs; on Linux wint_t is + // unsigned and this was already handled by #5348) + std::wstring w = L"[1]"; + w.push_back(static_cast(-1)); + w += L"garbage"; + CHECK(!json::accept(w)); + CHECK_THROWS_WITH_AS(_ = json::parse(w), + "[json.exception.parse_error.101] parse error at line 1, column 4: syntax error while parsing value - invalid literal; last read: '1]\xFF'; expected end of input", json::parse_error&); + + // other negative wchar_t units must not be truncated to their low + // byte (0xFFFFFF72 truncates to 'r', as in the u16string case above) + CHECK_THROWS_WITH_AS(_ = json::parse(std::wstring{L't', static_cast(0xFFFFFF72), L'u', L'e'}), + "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid literal; last read: 't\xFF'", json::parse_error&); +#endif + } } From 1a77948c25eee071319f2f07751455f884e25859 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 4 Oct 2026 11:59:34 +0200 Subject: [PATCH 17/22] Document the macros that preview version 4.0 in the roadmap (#5593) * Document the macros that preview version 4.0 in the roadmap List the macros that guard breaking changes planned to become the default in version 4.0.0, and explain that 4.0.0 will be the sum of these opt-in flags, which can be tried on the 3.x release train. Signed-off-by: Niels Lohmann * List the deprecated functions removed in 4.0 in the roadmap The roadmap lists what will be removed; the migration guide keeps the examples for how to replace each item. Also mention the deprecated (ptr, len) overloads of the from_* functions in the migration guide. Signed-off-by: Niels Lohmann * Wrap overlong line in cbor_tag_handler_t documentation The line added in #5559 exceeds the 160-character limit enforced by the documentation style check. Signed-off-by: Niels Lohmann * Add JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON to the 4.0 example Signed-off-by: Niels Lohmann * Add JSON_STRICT_BINARY_UTF8 to the 4.0 roadmap The macro comes from #5741: the CBOR, UBJSON, BJData, and BSON writers keep writing ill-formed UTF-8 unchanged in 3.x and are planned to check it by default in 4.0. Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann --- .../macros/json_brace_init_copy_semantics.md | 1 + docs/mkdocs/docs/community/roadmap.md | 70 +++++++++++++++++-- .../docs/integration/migration_guide.md | 7 +- 3 files changed, 71 insertions(+), 7 deletions(-) diff --git a/docs/mkdocs/docs/api/macros/json_brace_init_copy_semantics.md b/docs/mkdocs/docs/api/macros/json_brace_init_copy_semantics.md index 459818c87..71b8bb28f 100644 --- a/docs/mkdocs/docs/api/macros/json_brace_init_copy_semantics.md +++ b/docs/mkdocs/docs/api/macros/json_brace_init_copy_semantics.md @@ -115,3 +115,4 @@ The default value is `0` (disabled — existing behavior is preserved). ## Version history - Added in version 3.13.0. +- Planned to become the default (with the macro removed) in version 4.0.0. diff --git a/docs/mkdocs/docs/community/roadmap.md b/docs/mkdocs/docs/community/roadmap.md index e8c407d3f..48977b295 100644 --- a/docs/mkdocs/docs/community/roadmap.md +++ b/docs/mkdocs/docs/community/roadmap.md @@ -14,7 +14,7 @@ work items are tracked in the [GitHub milestones](https://github.com/nlohmann/js opt-in. - **Keep the 3.x public API stable.** Releases follow [semantic versioning](https://semver.org). Changes that would break existing code are only added behind a feature macro, so users can opt in and test their code before a next - major release. + major release, see [Version 4.0](#version-40). - **Support a broad range of compilers and platforms.** The [CI](quality_assurance.md) keeps testing old and new versions of GCC, Clang, MSVC, and other compilers on Linux, macOS, and Windows. - **Keep the quality assurance up.** Every change keeps the test coverage at 100%, passes the static and dynamic @@ -37,7 +37,67 @@ work items are tracked in the [GitHub milestones](https://github.com/nlohmann/js ## Version 4.0 -There is no decision yet on whether or when a version 4.0 with breaking changes will be released. Proposals that need -a major version, for instance stricter type conversions, are collected in issue -[#3453](https://github.com/nlohmann/json/issues/3453). Until then, such changes are only added as opt-in behavior -behind feature macros. +There is no release date for version 4.0 yet. Proposals that need a major version, for instance stricter type +conversions, are collected in issue [#3453](https://github.com/nlohmann/json/issues/3453). + +!!! note "Not final" + + The plan for version 4.0 described below is not final and may still change: macros may be added to or removed from + the list, and planned defaults may be revised. Any such change will be documented on this page. + +### Trying out 4.0 today + +Version 4.0 will not be developed on a separate branch. Instead, every breaking change is first added to a 3.x release +behind a macro whose default keeps the 3.x behavior. Version 4.0 then switches the defaults and removes the macros. +Version 4.0 is therefore the sum of these macros: you can try it on the 3.x release train today by defining each macro +to its 4.0 value and fixing what no longer compiles or behaves differently. Once your code works with all of them, it +is ready for version 4.0. + +The following macros guard changes that are planned to become the default in version 4.0: + +| Macro | 3.x default | 4.0 behavior | CMake option | Added | +|------------------------------------------------------------------------------------------------------------------|-------------|-------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------|--------| +| [`JSON_USE_IMPLICIT_CONVERSIONS`](../api/macros/json_use_implicit_conversions.md) | `1` | `0`: no implicit conversions from `basic_json` to other types; use [`get`](../api/basic_json/get.md) instead | [`JSON_ImplicitConversions`](../integration/cmake.md#json_implicitconversions) | 3.9.0 | +| [`JSON_USE_GLOBAL_UDLS`](../api/macros/json_use_global_udls.md) | `1` | `0`: the string literals `_json` and `_json_pointer` are only available in namespace `nlohmann::literals` | [`JSON_GlobalUDLs`](../integration/cmake.md#json_globaludls) | 3.11.0 | +| [`JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON`](../api/macros/json_use_legacy_discarded_value_comparison.md) | `0` | removed: the deprecated legacy comparison of discarded values can no longer be enabled | [`JSON_LegacyDiscardedValueComparison`](../integration/cmake.md#json_legacydiscardedvaluecomparison) | 3.11.0 | +| [`JSON_BRACE_INIT_COPY_SEMANTICS`](../api/macros/json_brace_init_copy_semantics.md) | `0` | `1`: single-element brace initialization such as `#!cpp json j{obj};` copies the element instead of creating an array | – | 3.13.0 | +| [`JSON_PRECISE_STREAM_POSITION`](../api/macros/json_precise_stream_position.md) | `0` | `1`: reading from a stream does not consume the character after a number | – | 3.13.0 | +| [`JSON_STRICT_NUL_HANDLING`](../api/macros/json_strict_nul_handling.md) | `0` | `1`: a NUL byte in the input is a parse error instead of the end of input | [`JSON_StrictNulHandling`](../integration/cmake.md#json_strictnulhandling) | 3.13.0 | +| [`JSON_STRICT_BINARY_UTF8`](../api/macros/json_strict_binary_utf8.md) | `0` | `1`: `to_cbor`, `to_ubjson`, `to_bjdata`, and `to_bson` throw for strings that are not valid UTF-8 by default | [`JSON_StrictBinaryUTF8`](../integration/cmake.md#json_strictbinaryutf8) | 3.13.0 | + +For example, the following makes a 3.x release behave like version 4.0 with respect to these changes: + +```cpp +#define JSON_USE_IMPLICIT_CONVERSIONS 0 +#define JSON_USE_GLOBAL_UDLS 0 +#define JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON 0 +#define JSON_BRACE_INIT_COPY_SEMANTICS 1 +#define JSON_PRECISE_STREAM_POSITION 1 +#define JSON_STRICT_NUL_HANDLING 1 +#define JSON_STRICT_BINARY_UTF8 1 +#include +``` + +The macros must be defined before the library header is included; setting them once in the build system is the easiest +way to achieve this. + +### Removal of deprecated functions + +Version 4.0 will remove all deprecated functions. Compiling with deprecation warnings enabled shows which of them your +code still uses. The [migration guide](../integration/migration_guide.md#replace-deprecated-functions) shows how to +replace each of them. + +| Deprecated | Since | Migration | +|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------|----------------------------------------------------------------------------------| +| `#!cpp operator<<(basic_json&, std::istream&)` | 3.0.0 | [Parsing](../integration/migration_guide.md#parsing) | +| `#!cpp operator>>(const basic_json&, std::ostream&)` | 3.0.0 | [Miscellaneous functions](../integration/migration_guide.md#miscellaneous-functions) | +| `iterator_wrapper` | 3.1.0 | [Miscellaneous functions](../integration/migration_guide.md#miscellaneous-functions) | +| [`parse`](../api/basic_json/parse.md), [`accept`](../api/basic_json/accept.md), and [`sax_parse`](../api/basic_json/sax_parse.md) with an initializer list `{ptr, len}` or `{first, last}` | 3.8.0 | [Parsing](../integration/migration_guide.md#parsing) | +| [`from_bson`](../api/basic_json/from_bson.md), [`from_cbor`](../api/basic_json/from_cbor.md), [`from_msgpack`](../api/basic_json/from_msgpack.md), and [`from_ubjson`](../api/basic_json/from_ubjson.md) with `(ptr, len)` or an initializer list | 3.8.0 | [Parsing](../integration/migration_guide.md#parsing) | +| [`json_pointer::operator string_t`](../api/json_pointer/operator_string_t.md) | 3.11.0 | [JSON Pointers](../integration/migration_guide.md#json-pointers) | +| [`json_pointer`](../api/json_pointer/index.md) with a `basic_json` type as template argument, and the overloads of `value`, `contains`, `operator[]`, and `at` accepting such a pointer | 3.11.0 | [JSON Pointers](../integration/migration_guide.md#json-pointers) | +| Comparing a [`json_pointer`](../api/json_pointer/index.md) with a string via [`operator==`](../api/json_pointer/operator_eq.md) or [`operator!=`](../api/json_pointer/operator_ne.md) | 3.11.2 | [JSON Pointers](../integration/migration_guide.md#json-pointers) | + +The deprecated legacy comparison of discarded values is controlled by a macro and therefore listed in the table above. + +New breaking changes will follow the same path: they are added to these tables when they land in a 3.x release. diff --git a/docs/mkdocs/docs/integration/migration_guide.md b/docs/mkdocs/docs/integration/migration_guide.md index 29d54b6df..8a07d48a3 100644 --- a/docs/mkdocs/docs/integration/migration_guide.md +++ b/docs/mkdocs/docs/integration/migration_guide.md @@ -2,11 +2,14 @@ This page collects some guidelines on how to future-proof your code for future versions of this library. For how to add the library to your project in the first place, see [Integration](index.md), [CMake](cmake.md), or -[Package Managers](package_managers.md). +[Package Managers](package_managers.md). The [roadmap](../community/roadmap.md#version-40) lists what will change in +version 4.0, including the macros that let you try its behavior with a 3.x release; this page describes how to adjust +your code. ## Replace deprecated functions -The following functions have been deprecated and will be removed in the next major version (i.e., 4.0.0). All +The following functions have been deprecated and will be removed in the next major version (i.e., 4.0.0), see the +[roadmap](../community/roadmap.md#removal-of-deprecated-functions) for an overview. All deprecations are annotated with [`HEDLEY_DEPRECATED_FOR`](https://nemequ.github.io/hedley/api-reference.html#HEDLEY_DEPRECATED_FOR) to report which function to use instead. From 40021f38fb306fc04b7b52173e8b2f6d1203c27d Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 4 Oct 2026 12:00:31 +0200 Subject: [PATCH 18/22] Add basic_json::as_base_class and document name conflicts with custom base classes (#5589) * Add basic_json::as_base_class and document name conflicts with custom base classes Members of basic_json hide members of a custom base class with the same name, and future releases may add members that hide ones accessible today. Document this in json_base_class_t and add as_base_class() to reach hidden members without spelling out the cast. Also make json_base_class_t a public member type. It was documented since 3.12.0, but declared private, so users could not name it. Supersedes #3899. Co-authored-by: Raphael Grimm <1005058+barcode@users.noreply.github.com> Signed-off-by: Niels Lohmann * Add as_base_class to the docset search index New public members get an entry in docs/docset/docSet.sql (as done for to_bon8/from_bon8 in #2998). Without it, the Dash/Zeal docset built from the documentation cannot find basic_json::as_base_class. Signed-off-by: Niels Lohmann * Silence clang-tidy for the hidden type_name() in the base class test ci_clang_tidy failed with readability-convert-member-functions-to-static on base_class_with_hidden_members::type_name(). It must stay a non-static member: the test shows that it is hidden by the non-static basic_json::type_name() and reachable through as_base_class(). Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann Co-authored-by: Raphael Grimm <1005058+barcode@users.noreply.github.com> --- docs/docset/docSet.sql | 1 + .../docs/api/basic_json/as_base_class.md | 53 ++++++++++++++ docs/mkdocs/docs/api/basic_json/index.md | 1 + .../docs/api/basic_json/json_base_class_t.md | 14 ++++ docs/mkdocs/docs/examples/as_base_class.cpp | 41 +++++++++++ .../mkdocs/docs/examples/as_base_class.output | 2 + docs/mkdocs/mkdocs.yml | 1 + include/nlohmann/json.hpp | 18 ++++- single_include/nlohmann/json.hpp | 18 ++++- tests/src/unit-custom-base-class.cpp | 70 +++++++++++++++++++ 10 files changed, 217 insertions(+), 2 deletions(-) create mode 100644 docs/mkdocs/docs/api/basic_json/as_base_class.md create mode 100644 docs/mkdocs/docs/examples/as_base_class.cpp create mode 100644 docs/mkdocs/docs/examples/as_base_class.output diff --git a/docs/docset/docSet.sql b/docs/docset/docSet.sql index 477b2f9fe..09802e43e 100644 --- a/docs/docset/docSet.sql +++ b/docs/docset/docSet.sql @@ -19,6 +19,7 @@ INSERT INTO searchIndex(name, type, path) VALUES ('format_as', 'Function', 'api/ INSERT INTO searchIndex(name, type, path) VALUES ('basic_json::accept', 'Function', 'api/basic_json/accept/index.html'); INSERT INTO searchIndex(name, type, path) VALUES ('basic_json::array', 'Function', 'api/basic_json/array/index.html'); INSERT INTO searchIndex(name, type, path) VALUES ('basic_json::array_t', 'Type', 'api/basic_json/array_t/index.html'); +INSERT INTO searchIndex(name, type, path) VALUES ('basic_json::as_base_class', 'Method', 'api/basic_json/as_base_class/index.html'); INSERT INTO searchIndex(name, type, path) VALUES ('basic_json::at', 'Method', 'api/basic_json/at/index.html'); INSERT INTO searchIndex(name, type, path) VALUES ('basic_json::back', 'Method', 'api/basic_json/back/index.html'); INSERT INTO searchIndex(name, type, path) VALUES ('basic_json::basic_json', 'Constructor', 'api/basic_json/basic_json/index.html'); diff --git a/docs/mkdocs/docs/api/basic_json/as_base_class.md b/docs/mkdocs/docs/api/basic_json/as_base_class.md new file mode 100644 index 000000000..769707140 --- /dev/null +++ b/docs/mkdocs/docs/api/basic_json/as_base_class.md @@ -0,0 +1,53 @@ +# nlohmann::basic_json::as_base_class + +```cpp +json_base_class_t& as_base_class() noexcept; +const json_base_class_t& as_base_class() const noexcept; +``` + +Returns a reference to this object as its custom base class [`json_base_class_t`](json_base_class_t.md). No copy is +made. + +Since `basic_json` derives from `json_base_class_t`, a member of `basic_json` hides any member of the custom base class +with the same name. This function makes such hidden members accessible again. + +## Return value + +reference to this object as [`json_base_class_t`](json_base_class_t.md) + +## Exception safety + +No-throw guarantee: this function never throws exceptions. + +## Complexity + +Constant. + +## Notes + +The function is equivalent to `static_cast(j)` (or `static_cast(j)`). + +## Examples + +??? example + + The example shows how to use `as_base_class` to access members of the custom base class that are hidden by members + of `basic_json`. + + ```cpp + --8<-- "examples/as_base_class.cpp" + ``` + + Output: + + ```json + --8<-- "examples/as_base_class.output" + ``` + +## See also + +- [json_base_class_t](json_base_class_t.md) - type of the custom base class + +## Version history + +- Added in version 3.13.0. diff --git a/docs/mkdocs/docs/api/basic_json/index.md b/docs/mkdocs/docs/api/basic_json/index.md index 866bda67a..fc934ca93 100644 --- a/docs/mkdocs/docs/api/basic_json/index.md +++ b/docs/mkdocs/docs/api/basic_json/index.md @@ -200,6 +200,7 @@ Direct access to the stored value of a JSON value. - [**get_ref**](get_ref.md) - get a reference value - [**operator ValueType**](operator_ValueType.md) - get a value - [**get_binary**](get_binary.md) - get a binary value +- [**as_base_class**](as_base_class.md) - access the custom base class ### Element access diff --git a/docs/mkdocs/docs/api/basic_json/json_base_class_t.md b/docs/mkdocs/docs/api/basic_json/json_base_class_t.md index 0d1abc9d4..6f0026558 100644 --- a/docs/mkdocs/docs/api/basic_json/json_base_class_t.md +++ b/docs/mkdocs/docs/api/basic_json/json_base_class_t.md @@ -27,6 +27,18 @@ A `CustomBaseClass` with non-static data members forfeits `basic_json`'s [standard layout](https://en.cppreference.com/w/cpp/named_req/StandardLayoutType) guarantee. See [Template Parameter Requirements](../../features/types/template_parameters.md#custombaseclass). +#### Name conflicts + +Since `basic_json` derives from `CustomBaseClass`, members of `basic_json` hide members of `CustomBaseClass` with the +same name. Hidden members remain accessible via [`as_base_class`](as_base_class.md) or by casting the value to +`json_base_class_t`. + +!!! warning "Avoid generic member names" + + Future versions of the library may add members to `basic_json` that hide members of `CustomBaseClass` that are + accessible today. To reduce the risk of such conflicts, avoid generic names for the members of `CustomBaseClass`, + for instance by using a distinctive prefix. + ## Examples ??? example @@ -45,8 +57,10 @@ A `CustomBaseClass` with non-static data members forfeits `basic_json`'s ## See also +- [as_base_class](as_base_class.md) - access the custom base class - [Template Parameter Requirements](../../features/types/template_parameters.md#custombaseclass) - the requirements for `CustomBaseClass` ## Version history - Added in version 3.12.0. +- Made a public member type in version 3.13.0; it was private before, so it could not be named outside the class. diff --git a/docs/mkdocs/docs/examples/as_base_class.cpp b/docs/mkdocs/docs/examples/as_base_class.cpp new file mode 100644 index 000000000..48357b045 --- /dev/null +++ b/docs/mkdocs/docs/examples/as_base_class.cpp @@ -0,0 +1,41 @@ +#include +#include + +class base_class_with_hidden_members +{ + public: + const char* type_name() const noexcept + { + return "my_type_name"; + } + + std::size_t size() const noexcept + { + return 42; + } +}; + +using json = nlohmann::basic_json < + std::map, + std::vector, + std::string, + bool, + std::int64_t, + std::uint64_t, + double, + std::allocator, + nlohmann::adl_serializer, + std::vector, + base_class_with_hidden_members + >; + +int main() +{ + json j = {1, 2, 3}; + + // the members of basic_json hide the members of the base class + std::cout << j.type_name() << ' ' << j.size() << '\n'; + + // access the hidden members of the base class + std::cout << j.as_base_class().type_name() << ' ' << j.as_base_class().size() << '\n'; +} diff --git a/docs/mkdocs/docs/examples/as_base_class.output b/docs/mkdocs/docs/examples/as_base_class.output new file mode 100644 index 000000000..5ca62a673 --- /dev/null +++ b/docs/mkdocs/docs/examples/as_base_class.output @@ -0,0 +1,2 @@ +array 3 +my_type_name 42 diff --git a/docs/mkdocs/mkdocs.yml b/docs/mkdocs/mkdocs.yml index b4daf34c3..7b5512084 100644 --- a/docs/mkdocs/mkdocs.yml +++ b/docs/mkdocs/mkdocs.yml @@ -116,6 +116,7 @@ nav: - 'accept': api/basic_json/accept.md - 'array': api/basic_json/array.md - 'array_t': api/basic_json/array_t.md + - 'as_base_class': api/basic_json/as_base_class.md - 'at': api/basic_json/at.md - 'back': api/basic_json/back.md - 'begin': api/basic_json/begin.md diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 89ba13233..07b625a89 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -162,7 +162,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// workaround type for MSVC using basic_json_t = NLOHMANN_BASIC_JSON_TPL; - using json_base_class_t = ::nlohmann::detail::json_base_class; JSON_PRIVATE_UNLESS_TESTED: // convenience aliases for types residing in namespace detail; @@ -216,6 +215,9 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec using cbor_tag_handler_t = detail::cbor_tag_handler_t; /// how to encode BJData using bjdata_version_t = detail::bjdata_version_t; + /// base class used to inject custom functionality into each instance of basic_json + /// @sa https://json.nlohmann.me/api/basic_json/json_base_class_t/ + using json_base_class_t = ::nlohmann::detail::json_base_class; /// helper type for initializer lists of basic_json values using initializer_list_t = std::initializer_list>; @@ -3023,6 +3025,20 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec return *get_ptr(); } + /// @brief access the custom base class + /// @sa https://json.nlohmann.me/api/basic_json/as_base_class/ + json_base_class_t& as_base_class() noexcept + { + return static_cast(*this); + } + + /// @brief access the custom base class + /// @sa https://json.nlohmann.me/api/basic_json/as_base_class/ + const json_base_class_t& as_base_class() const noexcept + { + return static_cast(*this); + } + /// @} private: diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 3b74fd67f..16e87cc86 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -26699,7 +26699,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// workaround type for MSVC using basic_json_t = NLOHMANN_BASIC_JSON_TPL; - using json_base_class_t = ::nlohmann::detail::json_base_class; JSON_PRIVATE_UNLESS_TESTED: // convenience aliases for types residing in namespace detail; @@ -26753,6 +26752,9 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec using cbor_tag_handler_t = detail::cbor_tag_handler_t; /// how to encode BJData using bjdata_version_t = detail::bjdata_version_t; + /// base class used to inject custom functionality into each instance of basic_json + /// @sa https://json.nlohmann.me/api/basic_json/json_base_class_t/ + using json_base_class_t = ::nlohmann::detail::json_base_class; /// helper type for initializer lists of basic_json values using initializer_list_t = std::initializer_list>; @@ -29560,6 +29562,20 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec return *get_ptr(); } + /// @brief access the custom base class + /// @sa https://json.nlohmann.me/api/basic_json/as_base_class/ + json_base_class_t& as_base_class() noexcept + { + return static_cast(*this); + } + + /// @brief access the custom base class + /// @sa https://json.nlohmann.me/api/basic_json/as_base_class/ + const json_base_class_t& as_base_class() const noexcept + { + return static_cast(*this); + } + /// @} private: diff --git a/tests/src/unit-custom-base-class.cpp b/tests/src/unit-custom-base-class.cpp index a6b9b9ea4..38b665793 100644 --- a/tests/src/unit-custom-base-class.cpp +++ b/tests/src/unit-custom-base-class.cpp @@ -10,6 +10,8 @@ #include #include #include +#include +#include #include #include "doctest_compatibility.h" @@ -406,6 +408,74 @@ TEST_CASE("JSON Visit Node") CHECK(expected.empty()); } +// Test accessing members of a custom base class that are hidden by members of nlohmann::basic_json +class base_class_with_hidden_members +{ + public: + const char* type_name() const noexcept // NOLINT(readability-convert-member-functions-to-static) + { + return "custom type_name"; + } + + std::size_t size() const noexcept + { + return m_size; + } + + std::size_t m_size = 42; +}; + +using json_with_hidden_base_members = + nlohmann::basic_json < + std::map, + std::vector, + std::string, + bool, + std::int64_t, + std::uint64_t, + double, + std::allocator, + nlohmann::adl_serializer, + std::vector, + base_class_with_hidden_members + >; + +TEST_CASE("JSON Node as_base_class") +{ + using json = json_with_hidden_base_members; + + static_assert(std::is_same().as_base_class()), json::json_base_class_t&>::value, ""); + static_assert(std::is_same().as_base_class()), const json::json_base_class_t&>::value, ""); + static_assert(noexcept(std::declval().as_base_class()), ""); + static_assert(noexcept(std::declval().as_base_class()), ""); + + SECTION("non-const") + { + json j = {1, 2, 3}; + + CHECK(std::string(j.type_name()) == "array"); + CHECK(j.size() == 3); + CHECK(std::string(j.as_base_class().type_name()) == "custom type_name"); + CHECK(j.as_base_class().size() == 42); + CHECK(&j.as_base_class() == &static_cast(j)); + + j.as_base_class().m_size = 7; + CHECK(j.as_base_class().size() == 7); + CHECK(j.size() == 3); + } + + SECTION("const") + { + const json j = {1, 2, 3}; + + CHECK(std::string(j.type_name()) == "array"); + CHECK(j.size() == 3); + CHECK(std::string(j.as_base_class().type_name()) == "custom type_name"); + CHECK(j.as_base_class().size() == 42); + CHECK(&j.as_base_class() == &static_cast(j)); + } +} + // A custom base class with a const member: copy-constructible (initializing a // const member works fine), but not copy-/move-assignable (assigning one does // not). Used to check that copy construction never requires more than that. From f56b418c56dcbf7c05fe6cca72e2a4de2d072b99 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 4 Oct 2026 12:13:48 +0200 Subject: [PATCH 19/22] Follow each binary format's UTF-8 rule: strict writers (CBOR/UBJSON/BJData/BSON), lenient readers (#5741) Signed-off-by: Niels Lohmann --- CMakeLists.txt | 6 + cmake/ci.cmake | 2 +- docs/mkdocs/docs/api/basic_json/to_bjdata.md | 7 +- docs/mkdocs/docs/api/basic_json/to_bson.md | 6 + docs/mkdocs/docs/api/basic_json/to_cbor.md | 8 ++ docs/mkdocs/docs/api/basic_json/to_ubjson.md | 5 + docs/mkdocs/docs/api/macros/index.md | 2 + .../api/macros/json_strict_binary_utf8.md | 97 +++++++++++++++ .../docs/features/binary_formats/bjdata.md | 16 +++ .../docs/features/binary_formats/bson.md | 16 +-- .../docs/features/binary_formats/cbor.md | 17 +-- .../features/binary_formats/messagepack.md | 15 +-- .../docs/features/binary_formats/ubjson.md | 16 +++ docs/mkdocs/docs/features/macros.md | 14 +++ docs/mkdocs/docs/features/namespace.md | 1 + docs/mkdocs/docs/home/exceptions.md | 13 ++- docs/mkdocs/docs/integration/cmake.md | 5 + docs/mkdocs/mkdocs.yml | 1 + include/nlohmann/detail/abi_macros.hpp | 19 ++- .../nlohmann/detail/input/binary_reader.hpp | 29 ++--- include/nlohmann/detail/macro_unscope.hpp | 1 + .../nlohmann/detail/output/binary_writer.hpp | 70 ++++++++++- include/nlohmann/detail/string_utils.hpp | 44 +------ single_include/nlohmann/json_fwd.hpp | 19 ++- tests/abi/config/default.cpp | 4 + tests/abi/config/noversion.cpp | 4 + tests/src/unit-binary_utf8_strict.cpp | 110 ++++++++++++++++++ tests/src/unit-bjdata.cpp | 37 ++++++ tests/src/unit-bson.cpp | 37 ++++++ tests/src/unit-cbor.cpp | 85 +++++++++++--- tests/src/unit-msgpack.cpp | 36 ++++-- tests/src/unit-ubjson.cpp | 37 ++++++ 32 files changed, 651 insertions(+), 128 deletions(-) create mode 100644 docs/mkdocs/docs/api/macros/json_strict_binary_utf8.md create mode 100644 tests/src/unit-binary_utf8_strict.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index f0a6771dc..b5092ae49 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -61,6 +61,7 @@ option(JSON_Install "Install CMake targets during install option(JSON_MultipleHeaders "Use non-amalgamated version of the library." ON) option(JSON_SystemInclude "Include as system headers (skip for clang-tidy)." OFF) option(JSON_StrictNulHandling "Build with strict NUL-byte handling enabled." OFF) +option(JSON_StrictBinaryUTF8 "Build with UTF-8 checks in the CBOR, UBJSON, BJData, and BSON writers enabled." OFF) if (JSON_CI) include(ci) @@ -118,6 +119,10 @@ if (JSON_StrictNulHandling) message(STATUS "Strict NUL-byte handling enabled (JSON_STRICT_NUL_HANDLING=1)") endif() +if (JSON_StrictBinaryUTF8) + message(STATUS "Strict UTF-8 checks in binary writers enabled (JSON_STRICT_BINARY_UTF8=1)") +endif() + if (JSON_Diagnostic_Positions) message(STATUS "Diagnostic positions enabled (JSON_DIAGNOSTIC_POSITIONS=1)") endif() @@ -153,6 +158,7 @@ target_compile_definitions( $<$:JSON_DIAGNOSTIC_POSITIONS=1> $<$:JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON=1> $<$:JSON_STRICT_NUL_HANDLING=1> + $<$:JSON_STRICT_BINARY_UTF8=1> ) target_include_directories( diff --git a/cmake/ci.cmake b/cmake/ci.cmake index e67aba6d3..2aceb5afd 100644 --- a/cmake/ci.cmake +++ b/cmake/ci.cmake @@ -701,7 +701,7 @@ ci_get_cmake(4.0.0 CMAKE_4_0_0_BINARY) # the tests require CMake 3.13 or later, so they are excluded for CMake 3.5.0 set(JSON_CMAKE_FLAGS_3_5_0 JSON_Diagnostics JSON_Diagnostic_Positions JSON_GlobalUDLs JSON_ImplicitConversions JSON_DisableEnumSerialization JSON_LegacyDiscardedValueComparison JSON_Install JSON_MultipleHeaders JSON_SystemInclude JSON_Valgrind - JSON_StrictNulHandling) + JSON_StrictNulHandling JSON_StrictBinaryUTF8) set(JSON_CMAKE_FLAGS_3_31_6 JSON_BuildTests ${JSON_CMAKE_FLAGS_3_5_0}) set(JSON_CMAKE_FLAGS_4_0_0 JSON_BuildTests ${JSON_CMAKE_FLAGS_3_5_0}) diff --git a/docs/mkdocs/docs/api/basic_json/to_bjdata.md b/docs/mkdocs/docs/api/basic_json/to_bjdata.md index 44cc399e1..63dc5379e 100644 --- a/docs/mkdocs/docs/api/basic_json/to_bjdata.md +++ b/docs/mkdocs/docs/api/basic_json/to_bjdata.md @@ -56,6 +56,9 @@ Strong guarantee: if an exception is thrown, there are no changes in the JSON va - Throws [`other_error.502`](../../home/exceptions.md#jsonexceptionother_error502) if `use_type` is true and `use_size` is false, and `j` contains a non-empty array, object, or binary value. +- Throws [type_error.316](../../home/exceptions.md#jsonexceptiontype_error316) if a string or object key in `j` is not + valid UTF-8 and [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled; otherwise, the bytes are + written unchanged ## Complexity @@ -104,4 +107,6 @@ Linear in the size of the JSON value `j`. ## Version history - Added in version 3.11.0. -- BJData version parameter (for draft3 binary encoding) added in version 3.12.0. \ No newline at end of file +- BJData version parameter (for draft3 binary encoding) added in version 3.12.0. +- Throwing `type_error.316` for a string or object key that is not valid UTF-8 if + [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled added in version 3.13.0. \ No newline at end of file diff --git a/docs/mkdocs/docs/api/basic_json/to_bson.md b/docs/mkdocs/docs/api/basic_json/to_bson.md index c79f39ffc..e3104b13c 100644 --- a/docs/mkdocs/docs/api/basic_json/to_bson.md +++ b/docs/mkdocs/docs/api/basic_json/to_bson.md @@ -46,6 +46,9 @@ Strong guarantee: if an exception is thrown, there are no changes in the JSON va - Throws [`out_of_range.415`](../../home/exceptions.md#jsonexceptionout_of_range415) if the subtype of a binary value exceeds 255, the maximum of the BSON binary subtype; example: `"subtype 70000 is too large for the BSON binary subtype (max 255)"` +- Throws [type_error.316](../../home/exceptions.md#jsonexceptiontype_error316) if a string or object key is not valid + UTF-8 and [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled; otherwise, the bytes are + written unchanged ## Complexity @@ -98,3 +101,6 @@ pass before anything is written. - Throws `out_of_range.412` and `out_of_range.415` since version 3.13.0. - Linear in the size of `j`, and no longer limited by the call stack for deeply nested values, since version 3.13.0. - `out_of_range.415` is now detected before anything is written, like the other exceptions above, since version 3.13.0. +- Throwing `type_error.316` for a string value or object key that is not valid UTF-8 if + [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled, detected before anything is written, + added in version 3.13.0. diff --git a/docs/mkdocs/docs/api/basic_json/to_cbor.md b/docs/mkdocs/docs/api/basic_json/to_cbor.md index 3bbd9c7d3..cac8a0917 100644 --- a/docs/mkdocs/docs/api/basic_json/to_cbor.md +++ b/docs/mkdocs/docs/api/basic_json/to_cbor.md @@ -35,6 +35,12 @@ The exact mapping and its limitations are described on a [dedicated page](../../ Strong guarantee: if an exception is thrown, there are no changes in the JSON value. +## Exceptions + +- Throws [type_error.316](../../home/exceptions.md#jsonexceptiontype_error316) if a string or object key in `j` is not + valid UTF-8 and [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled; otherwise, the bytes are + written unchanged + ## Complexity Linear in the size of the JSON value `j`. @@ -68,3 +74,5 @@ Linear in the size of the JSON value `j`. - Added in version 2.0.9. - Compact representation of floating-point numbers added in version 3.8.0. +- Throwing `type_error.316` for a string or object key that is not valid UTF-8 if + [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled added in version 3.13.0. diff --git a/docs/mkdocs/docs/api/basic_json/to_ubjson.md b/docs/mkdocs/docs/api/basic_json/to_ubjson.md index 1b7f7767e..8f2ea5eb9 100644 --- a/docs/mkdocs/docs/api/basic_json/to_ubjson.md +++ b/docs/mkdocs/docs/api/basic_json/to_ubjson.md @@ -49,6 +49,9 @@ Strong guarantee: if an exception is thrown, there are no changes in the JSON va - Throws [`other_error.502`](../../home/exceptions.md#jsonexceptionother_error502) if `use_type` is true and `use_size` is false, and `j` contains a non-empty array, object, or binary value. +- Throws [type_error.316](../../home/exceptions.md#jsonexceptiontype_error316) if a string or object key in `j` is not + valid UTF-8 and [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled; otherwise, the bytes are + written unchanged ## Complexity @@ -97,3 +100,5 @@ Linear in the size of the JSON value `j`. ## Version history - Added in version 3.1.0. +- Throwing `type_error.316` for a string or object key that is not valid UTF-8 if + [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled added in version 3.13.0. diff --git a/docs/mkdocs/docs/api/macros/index.md b/docs/mkdocs/docs/api/macros/index.md index 3c9b42af2..872d6cb8c 100644 --- a/docs/mkdocs/docs/api/macros/index.md +++ b/docs/mkdocs/docs/api/macros/index.md @@ -18,6 +18,8 @@ header. See also the [macro overview page](../../features/macros.md). - [**JSON_PRECISE_STREAM_POSITION**](json_precise_stream_position.md) - opt in to leaving an input stream positioned right after a parsed number +- [**JSON_STRICT_BINARY_UTF8**](json_strict_binary_utf8.md) - opt in to checking strings for valid UTF-8 in the CBOR, + UBJSON, BJData, and BSON writers - [**JSON_STRICT_NUL_HANDLING**](json_strict_nul_handling.md) - opt in to rejecting a NUL byte in the input instead of treating it as end of input diff --git a/docs/mkdocs/docs/api/macros/json_strict_binary_utf8.md b/docs/mkdocs/docs/api/macros/json_strict_binary_utf8.md new file mode 100644 index 000000000..7fe6d9a58 --- /dev/null +++ b/docs/mkdocs/docs/api/macros/json_strict_binary_utf8.md @@ -0,0 +1,97 @@ +# JSON_STRICT_BINARY_UTF8 + +```cpp +#define JSON_STRICT_BINARY_UTF8 /* value */ +``` + +When defined to `1`, the binary writers [`to_cbor`](../basic_json/to_cbor.md), [`to_ubjson`](../basic_json/to_ubjson.md), +[`to_bjdata`](../basic_json/to_bjdata.md), and [`to_bson`](../basic_json/to_bson.md) check every string value and +object key for valid UTF-8 and throw [`type_error.316`](../../home/exceptions.md#jsonexceptiontype_error316) for +ill-formed UTF-8, like [`dump`](../basic_json/dump.md) does. Without it, they write the bytes unchanged. + +The macro does not affect: + +- [`to_msgpack`](../basic_json/to_msgpack.md): the MessagePack specification allows a `str` value to contain bytes that + are not valid UTF-8, so it always writes them unchanged. +- [`to_bon8`](../basic_json/to_bon8.md): BON8 always checks, because the UTF-8 lead bytes mark where a string ends. +- The binary readers ([`from_cbor`](../basic_json/from_cbor.md), [`from_msgpack`](../basic_json/from_msgpack.md), + [`from_ubjson`](../basic_json/from_ubjson.md), [`from_bjdata`](../basic_json/from_bjdata.md), + [`from_bson`](../basic_json/from_bson.md)): none of these formats requires a decoder to reject ill-formed UTF-8, so + they always return the bytes unchanged. + +## Default definition + +The default value is `0` (disabled, the behavior of version 3.12.0 and earlier is preserved). + +```cpp +#define JSON_STRICT_BINARY_UTF8 0 +``` + +## Notes + +!!! note "Background" + + CBOR, UBJSON, BJData, and BSON all require strings to be UTF-8. Up to version 3.12.0, the writers did not check + this, so they could produce output that other decoders reject. Checking by default would break code that stores + other encodings (for instance ISO 8859-1) in a string and only ever writes it to a binary format, so this macro + offers the check as an opt-in ahead of version 4.0.0, where it is planned to become the default (see + [#5529](https://github.com/nlohmann/json/issues/5529) and [#5651](https://github.com/nlohmann/json/issues/5651)). + +!!! warning "Opt-in only" + + This macro must be defined **before** including ``. Defining it after the include has no + effect. + +!!! note "ABI compatibility" + + The value of this macro is encoded in the [namespace](../../features/namespace.md) (tag `_sbu8`), resulting in + distinct symbol names. Translation units compiled with and without it can therefore be linked into the same program + without One Definition Rule (ODR) violations, but they cannot exchange instances of library types. + +## Examples + +??? example "Default behavior (macro not defined)" + + Without the macro, the bytes are written unchanged: + + ```cpp + #include + + using json = nlohmann::json; + + int main() + { + auto v = json::to_cbor(json("\xFF")); + // v is {0x61, 0xFF} + } + ``` + +??? example "Opt-in check (macro defined to 1)" + + With the macro, ill-formed UTF-8 is rejected: + + ```cpp + #define JSON_STRICT_BINARY_UTF8 1 + #include + + using json = nlohmann::json; + + int main() + { + auto v = json::to_cbor(json("\xFF")); + // throws type_error.316: invalid UTF-8 byte at index 0: 0xFF + } + ``` + +## See also + +- [**to_cbor**](../basic_json/to_cbor.md) - create a CBOR serialization of a JSON value +- [**to_ubjson**](../basic_json/to_ubjson.md) - create a UBJSON serialization of a JSON value +- [**to_bjdata**](../basic_json/to_bjdata.md) - create a BJData serialization of a JSON value +- [**to_bson**](../basic_json/to_bson.md) - create a BSON serialization of a JSON value +- [**error_handler_t**](../basic_json/error_handler_t.md) - how [`dump`](../basic_json/dump.md) treats ill-formed UTF-8 + +## Version history + +- Added in version 3.13.0. +- Planned to become the default (with the macro removed) in version 4.0.0. diff --git a/docs/mkdocs/docs/features/binary_formats/bjdata.md b/docs/mkdocs/docs/features/binary_formats/bjdata.md index 8e3ed41f6..12d1af51b 100644 --- a/docs/mkdocs/docs/features/binary_formats/bjdata.md +++ b/docs/mkdocs/docs/features/binary_formats/bjdata.md @@ -63,6 +63,13 @@ The library uses the following mapping from JSON values types to BJData types ac - strings with more than 18446744073709551615 bytes, i.e., 264-1 bytes (theoretical) +!!! warning "UTF-8 validation of string values and object keys" + + BJData strings must use UTF-8 encoding. By default, `to_bjdata()` writes the bytes of string values and object keys + unchanged, even if they are not valid UTF-8. If + [`JSON_STRICT_BINARY_UTF8`](../../api/macros/json_strict_binary_utf8.md) is enabled, it throws + [`type_error.316`](../../home/exceptions.md#jsonexceptiontype_error316) for ill-formed UTF-8 instead. + !!! info "Unused BJData markers" The following markers are not used in the conversion: @@ -208,6 +215,15 @@ The library maps BJData types to JSON value types as follows: The mapping is **complete** in the sense that any BJData value can be converted to a JSON value. +!!! warning "Ill-formed UTF-8 in string values and object keys" + + BJData strings must use UTF-8 encoding, but this is not enforced on read: `from_bjdata()` accepts a string + value or object key whose bytes are not valid UTF-8 and hands them back unchanged. However, + [`dump()`](../../api/basic_json/dump.md) still requires valid UTF-8 and throws + [`type_error.316`](../../home/exceptions.md#jsonexceptiontype_error316) for such a value, unless an error + handler is passed that replaces or ignores the ill-formed bytes. By default, `to_bjdata()` writes such a value + back unchanged (see above). + !!! info "Round trips" A value returned by [`from_bjdata`](../../api/basic_json/from_bjdata.md) can be serialized with diff --git a/docs/mkdocs/docs/features/binary_formats/bson.md b/docs/mkdocs/docs/features/binary_formats/bson.md index cca11451e..245ed3ebf 100644 --- a/docs/mkdocs/docs/features/binary_formats/bson.md +++ b/docs/mkdocs/docs/features/binary_formats/bson.md @@ -109,14 +109,16 @@ The library maps BSON record types to JSON value types as follows: If BSON input must be validated for strict specification compliance, validate it separately before passing it to `from_bson()`. -!!! warning "UTF-8 validation of string values" +!!! warning "Ill-formed UTF-8 in string values" - The BSON specification requires `string` values (type `0x02`) to be valid UTF-8. This library validates the - bytes of every such string at decode time and rejects ill-formed UTF-8 with a - [`parse_error.113`](../../home/exceptions.md#jsonexceptionparse_error113) exception (or, with `allow_exceptions` - set to `false`, a discarded value), rather than only failing later when the resulting value is dumped. Element - (key) names and `binary` values (type `0x05`) are unaffected and are never validated, since they are read - byte-by-byte as a C string, or are not required to hold text, respectively. + The BSON specification requires `string` values (type `0x02`) to be valid UTF-8, but this is not required of a + decoder. `from_bson()` accepts a `string` value whose bytes are not valid UTF-8 and hands them back unchanged. + However, [`dump()`](../../api/basic_json/dump.md) still requires valid UTF-8 and throws + [`type_error.316`](../../home/exceptions.md#jsonexceptiontype_error316) for such a value, unless an error handler is + passed that replaces or ignores the ill-formed bytes. By default, `to_bson()` writes such a string value or element + (key) name unchanged; if [`JSON_STRICT_BINARY_UTF8`](../../api/macros/json_strict_binary_utf8.md) is enabled, it + throws the same exception instead. Element (key) names are never validated on read, since they are read byte-by-byte + as a C string. `binary` values (type `0x05`) are unaffected, since they are not required to hold text. ??? example "Example: deserialize a JSON value from BSON" diff --git a/docs/mkdocs/docs/features/binary_formats/cbor.md b/docs/mkdocs/docs/features/binary_formats/cbor.md index eb7bc7f41..66300c5fa 100644 --- a/docs/mkdocs/docs/features/binary_formats/cbor.md +++ b/docs/mkdocs/docs/features/binary_formats/cbor.md @@ -189,15 +189,16 @@ The library maps CBOR types to JSON value types as follows: ([RFC 8392](https://www.rfc-editor.org/rfc/rfc8392.html)), cannot be read with this library and need a general-purpose CBOR library instead. -!!! warning "UTF-8 validation of text strings" +!!! warning "Ill-formed UTF-8 in text strings" - [RFC 8949, Section 3.1](https://www.rfc-editor.org/rfc/rfc8949.html#section-3.1) requires CBOR text strings - (major type 3) to be valid UTF-8. This library validates the bytes of every text string (object keys included) at - decode time and rejects ill-formed UTF-8 with a - [`parse_error.113`](../../home/exceptions.md#jsonexceptionparse_error113) exception (or, with - `allow_exceptions` set to `false`, a discarded value), rather than only failing later when the resulting value is - dumped. Byte strings (major type 2) are unaffected and are never validated, since they are not required to hold - text. + [RFC 8949, Section 3.1](https://www.rfc-editor.org/rfc/rfc8949.html#section-3.1) requires CBOR text strings (major + type 3) to be valid UTF-8, but leaves it up to the decoder whether to enforce this. This library does not: + `from_cbor()` accepts a text string (object keys included) whose bytes are not valid UTF-8 and hands them back + unchanged. However, [`dump()`](../../api/basic_json/dump.md) still requires valid UTF-8 and throws + [`type_error.316`](../../home/exceptions.md#jsonexceptiontype_error316) for such a value, unless an error handler is + passed that replaces or ignores the ill-formed bytes. By default, `to_cbor()` writes such a value back unchanged; if + [`JSON_STRICT_BINARY_UTF8`](../../api/macros/json_strict_binary_utf8.md) is enabled, it throws the same exception + instead. Byte strings (major type 2) are unaffected, since they are not required to hold text. !!! warning "Tagged items" diff --git a/docs/mkdocs/docs/features/binary_formats/messagepack.md b/docs/mkdocs/docs/features/binary_formats/messagepack.md index 2e674252a..047944852 100644 --- a/docs/mkdocs/docs/features/binary_formats/messagepack.md +++ b/docs/mkdocs/docs/features/binary_formats/messagepack.md @@ -153,14 +153,15 @@ The library maps MessagePack types to JSON value types as follows: This applies to the [SAX interface](../parsing/sax_interface.md) as well, as the key is read before it is passed on. Such input needs a general-purpose MessagePack library instead. -!!! warning "UTF-8 validation of string values" +!!! warning "Ill-formed UTF-8 in string values" - The MessagePack specification requires `str` values (`fixstr`, `str 8`, `str 16`, `str 32`) to be valid UTF-8. - This library validates the bytes of every such string (object keys included) at decode time and rejects - ill-formed UTF-8 with a [`parse_error.113`](../../home/exceptions.md#jsonexceptionparse_error113) exception (or, - with `allow_exceptions` set to `false`, a discarded value), rather than only failing later when the resulting - value is dumped. `bin`/`ext`/`fixext` values are unaffected and are never validated, since they are not required - to hold text. + The MessagePack specification explicitly allows a `str` value (`fixstr`, `str 8`, `str 16`, `str 32`) to contain + a byte sequence that is not valid UTF-8, and expects a deserializer to hand the original bytes back unchanged. + This library follows that: `from_msgpack()` reads `str` bytes (object keys included) as-is, without validating + them, and `to_msgpack()` writes them back as-is, so such a value round-trips through `from_msgpack(to_msgpack(j))` + byte for byte. However, [`dump()`](../../api/basic_json/dump.md) still requires valid UTF-8 and throws + [`type_error.316`](../../home/exceptions.md#jsonexceptiontype_error316) for a value read this way, unless an + error handler is passed that replaces or ignores the ill-formed bytes. ??? example "Example: deserialize a JSON value from MessagePack" diff --git a/docs/mkdocs/docs/features/binary_formats/ubjson.md b/docs/mkdocs/docs/features/binary_formats/ubjson.md index 37aa069e2..dbdff6e6c 100644 --- a/docs/mkdocs/docs/features/binary_formats/ubjson.md +++ b/docs/mkdocs/docs/features/binary_formats/ubjson.md @@ -47,6 +47,13 @@ The library uses the following mapping from JSON values types to UBJSON types ac - strings with more than 9223372036854775807 bytes (theoretical) +!!! warning "UTF-8 validation of string values and object keys" + + UBJSON's required string encoding is UTF-8. By default, `to_ubjson()` writes the bytes of string values and object + keys unchanged, even if they are not valid UTF-8. If + [`JSON_STRICT_BINARY_UTF8`](../../api/macros/json_strict_binary_utf8.md) is enabled, it throws + [`type_error.316`](../../home/exceptions.md#jsonexceptiontype_error316) for ill-formed UTF-8 instead. + !!! info "Unused UBJSON markers" The following markers are not used in the conversion: @@ -120,6 +127,15 @@ The library maps UBJSON types to JSON value types as follows: The mapping is **complete** in the sense that any UBJSON value can be converted to a JSON value. +!!! warning "Ill-formed UTF-8 in string values and object keys" + + UBJSON's required string encoding is UTF-8, but this is not enforced on read: `from_ubjson()` accepts a string + value or object key whose bytes are not valid UTF-8 and hands them back unchanged. However, + [`dump()`](../../api/basic_json/dump.md) still requires valid UTF-8 and throws + [`type_error.316`](../../home/exceptions.md#jsonexceptiontype_error316) for such a value, unless an error + handler is passed that replaces or ignores the ill-formed bytes. By default, `to_ubjson()` writes such a value + back unchanged (see above). + ??? example "Example: deserialize a JSON value from UBJSON" ```cpp diff --git a/docs/mkdocs/docs/features/macros.md b/docs/mkdocs/docs/features/macros.md index 90bb352c3..681980315 100644 --- a/docs/mkdocs/docs/features/macros.md +++ b/docs/mkdocs/docs/features/macros.md @@ -138,6 +138,20 @@ using the library with compilers that do not fully support C++11 and may only wo See [full documentation of `JSON_SKIP_UNSUPPORTED_COMPILER_CHECK`](../api/macros/json_skip_unsupported_compiler_check.md). +## `JSON_STRICT_BINARY_UTF8` + +When defined to `1`, [`to_cbor`](../api/basic_json/to_cbor.md), [`to_ubjson`](../api/basic_json/to_ubjson.md), +[`to_bjdata`](../api/basic_json/to_bjdata.md), and [`to_bson`](../api/basic_json/to_bson.md) throw +[`type_error.316`](../home/exceptions.md#jsonexceptiontype_error316) for a string value or object key that is not +valid UTF-8. The default value is `0`, which writes the bytes unchanged as before version 3.13.0; this is planned to +become the default in version 4.0.0. + +The check can also be enabled with the CMake option +[`JSON_StrictBinaryUTF8`](../integration/cmake.md#json_strictbinaryutf8) (`OFF` by default) which sets +`JSON_STRICT_BINARY_UTF8` accordingly. + +See [full documentation of `JSON_STRICT_BINARY_UTF8`](../api/macros/json_strict_binary_utf8.md). + ## `JSON_STRICT_NUL_HANDLING` When defined to `1`, a `'\0'` (NUL) byte anywhere in the input is rejected with `parse_error.101`, like any other diff --git a/docs/mkdocs/docs/features/namespace.md b/docs/mkdocs/docs/features/namespace.md index 577f5e221..dbddac13d 100644 --- a/docs/mkdocs/docs/features/namespace.md +++ b/docs/mkdocs/docs/features/namespace.md @@ -20,6 +20,7 @@ The complete default namespace name is derived as follows: `_bics`. - [`JSON_PRECISE_STREAM_POSITION`](../api/macros/json_precise_stream_position.md) defined non-zero appends `_psp`. - [`JSON_STRICT_NUL_HANDLING`](../api/macros/json_strict_nul_handling.md) defined non-zero appends `_snul`. + - [`JSON_STRICT_BINARY_UTF8`](../api/macros/json_strict_binary_utf8.md) defined non-zero appends `_sbu8`. - The inline namespace ends with the suffix `_v` followed by the 3 components of the version number separated by underscores. To omit the version component, see [Disabling the version component](#disabling-the-version-component) below. diff --git a/docs/mkdocs/docs/home/exceptions.md b/docs/mkdocs/docs/home/exceptions.md index c78aeaa68..a3797cf04 100644 --- a/docs/mkdocs/docs/home/exceptions.md +++ b/docs/mkdocs/docs/home/exceptions.md @@ -340,8 +340,9 @@ An unexpected byte was read in a [binary format](../features/binary_formats/inde ### json.exception.parse_error.113 A string could not be read from a [binary format](../features/binary_formats/index.md): either a value that is not a -string was read where one was required (for instance as a map key), the string's length specification is invalid, or -the string's bytes are not valid UTF-8. +string was read where one was required (for instance as a map key), or the string's length specification is invalid. +The bytes of a string itself are not checked for valid UTF-8 on read; see the ill-formed UTF-8 notes on the +individual [binary format](../features/binary_formats/index.md) pages for how such a string is handled afterward. CBOR and MessagePack allow map keys of any type, but JSON object keys are always strings. Maps with keys of any other type (for instance integers or `null`) are therefore not supported; see the notes on @@ -364,9 +365,6 @@ type (for instance integers or `null`) are therefore not supported; see the note ``` [json.exception.parse_error.113] parse error at byte 3: syntax error while parsing BJData string: string length must not be negative ``` - ``` - [json.exception.parse_error.113] parse error at byte 3: syntax error while parsing CBOR string: invalid string: ill-formed UTF-8 byte - ``` ### json.exception.parse_error.114 @@ -749,6 +747,11 @@ The [`unflatten()`](../api/basic_json/unflatten.md) function only works for an o The [`dump()`](../api/basic_json/dump.md) function only works with UTF-8 encoded strings; that is, if you assign a `std::string` to a JSON value, make sure it is UTF-8 encoded. See the FAQ entry on [serializing untrusted or invalid UTF-8](faq.md#serializing-untrusted-or-invalid-utf-8) for background and the recommended fix. +If [`JSON_STRICT_BINARY_UTF8`](../api/macros/json_strict_binary_utf8.md) is enabled, the binary writers +[`to_cbor()`](../api/basic_json/to_cbor.md), [`to_ubjson()`](../api/basic_json/to_ubjson.md), +[`to_bjdata()`](../api/basic_json/to_bjdata.md), and [`to_bson()`](../api/basic_json/to_bson.md) throw this exception +for a string value or object key that is not valid UTF-8 as well. + !!! failure "Example message" Calling `dump()` on a JSON value containing an ISO 8859-1 encoded string: diff --git a/docs/mkdocs/docs/integration/cmake.md b/docs/mkdocs/docs/integration/cmake.md index 785fc3ed6..9151d12be 100644 --- a/docs/mkdocs/docs/integration/cmake.md +++ b/docs/mkdocs/docs/integration/cmake.md @@ -212,6 +212,11 @@ Use the non-amalgamated version of the library. This option is `ON` by default. Treat the library headers like system headers (i.e., adding `SYSTEM` to the [`target_include_directories`](https://cmake.org/cmake/help/latest/command/target_include_directories.html) call) to check for this library by tools like Clang-Tidy. This option is `OFF` by default. +### `JSON_StrictBinaryUTF8` + +Check string values and object keys for valid UTF-8 in the CBOR, UBJSON, BJData, and BSON writers, by defining the +macro [`JSON_STRICT_BINARY_UTF8`](../api/macros/json_strict_binary_utf8.md). This option is `OFF` by default. + ### `JSON_StrictNulHandling` Reject a `'\0'` (NUL) byte in the input instead of treating it as end of input, by defining the macro diff --git a/docs/mkdocs/mkdocs.yml b/docs/mkdocs/mkdocs.yml index 7b5512084..3cb25ba9b 100644 --- a/docs/mkdocs/mkdocs.yml +++ b/docs/mkdocs/mkdocs.yml @@ -307,6 +307,7 @@ nav: - 'JSON_PRECISE_STREAM_POSITION': api/macros/json_precise_stream_position.md - 'JSON_SKIP_LIBRARY_VERSION_CHECK': api/macros/json_skip_library_version_check.md - 'JSON_SKIP_UNSUPPORTED_COMPILER_CHECK': api/macros/json_skip_unsupported_compiler_check.md + - 'JSON_STRICT_BINARY_UTF8': api/macros/json_strict_binary_utf8.md - 'JSON_STRICT_NUL_HANDLING': api/macros/json_strict_nul_handling.md - 'JSON_USE_GLOBAL_UDLS': api/macros/json_use_global_udls.md - 'JSON_USE_IMPLICIT_CONVERSIONS': api/macros/json_use_implicit_conversions.md diff --git a/include/nlohmann/detail/abi_macros.hpp b/include/nlohmann/detail/abi_macros.hpp index 0bace616a..0153c8706 100644 --- a/include/nlohmann/detail/abi_macros.hpp +++ b/include/nlohmann/detail/abi_macros.hpp @@ -46,6 +46,10 @@ #define JSON_STRICT_NUL_HANDLING 0 #endif +#ifndef JSON_STRICT_BINARY_UTF8 + #define JSON_STRICT_BINARY_UTF8 0 +#endif + #if JSON_DIAGNOSTICS #define NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS _diag #else @@ -82,14 +86,20 @@ #define NLOHMANN_JSON_ABI_TAG_STRICT_NUL_HANDLING #endif +#if JSON_STRICT_BINARY_UTF8 + #define NLOHMANN_JSON_ABI_TAG_STRICT_BINARY_UTF8 _sbu8 +#else + #define NLOHMANN_JSON_ABI_TAG_STRICT_BINARY_UTF8 +#endif + #ifndef NLOHMANN_JSON_NAMESPACE_NO_VERSION #define NLOHMANN_JSON_NAMESPACE_NO_VERSION 0 #endif // Construct the namespace ABI tags component -#define NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b, c, d, e, f) json_abi ## a ## b ## c ## d ## e ## f -#define NLOHMANN_JSON_ABI_TAGS_CONCAT(a, b, c, d, e, f) \ - NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b, c, d, e, f) +#define NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b, c, d, e, f, g) json_abi ## a ## b ## c ## d ## e ## f ## g +#define NLOHMANN_JSON_ABI_TAGS_CONCAT(a, b, c, d, e, f, g) \ + NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b, c, d, e, f, g) #define NLOHMANN_JSON_ABI_TAGS \ NLOHMANN_JSON_ABI_TAGS_CONCAT( \ @@ -98,7 +108,8 @@ NLOHMANN_JSON_ABI_TAG_DIAGNOSTIC_POSITIONS, \ NLOHMANN_JSON_ABI_TAG_BRACE_INIT_COPY_SEMANTICS, \ NLOHMANN_JSON_ABI_TAG_PRECISE_STREAM_POSITION, \ - NLOHMANN_JSON_ABI_TAG_STRICT_NUL_HANDLING) + NLOHMANN_JSON_ABI_TAG_STRICT_NUL_HANDLING, \ + NLOHMANN_JSON_ABI_TAG_STRICT_BINARY_UTF8) // Construct the namespace version component #define NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT_EX(major, minor, patch) \ diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index 86f00bef6..3d79ad41b 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -4044,28 +4044,13 @@ class binary_reader const NumberType len, string_t& result) { - // get_bytes() appends to result, and CBOR indefinite-length strings - // collect all their chunks in the same result; validating only the - // newly read bytes keeps the check linear in the input size - const std::size_t old_size = result.size(); - if (JSON_HEDLEY_UNLIKELY(!get_bytes(format, len, "string", result))) - { - return false; - } - - // RFC 8949 (CBOR) §3.1 and the MessagePack/BSON/UBJSON specifications - // all require text strings to be valid UTF-8; reject anything else - // right here so malformed input is caught at decode time instead of - // only surfacing later as a type_error.316 when the value is dumped - // (which would defeat allow_exceptions=false / strict discarding). - if (JSON_HEDLEY_UNLIKELY(!is_valid_utf8(result, old_size))) - { - return sax->parse_error(chars_read, get_token_string(), - parse_error::create(113, chars_read, - exception_message(format, "invalid string: ill-formed UTF-8 byte", "string"), nullptr)); - } - - return true; + // Strings are taken as is: none of CBOR (RFC 8949 §3.1 leaves the + // choice to the decoder), MessagePack (whose spec explicitly allows + // a str object to contain an invalid byte sequence), UBJSON, BJData, + // or BSON requires a decoder to reject ill-formed UTF-8. The bytes + // are kept unchanged; dump() and the binary writers are the ones + // that check them and report type_error.316 if they are not valid. + return get_bytes(format, len, "string", result); } /*! diff --git a/include/nlohmann/detail/macro_unscope.hpp b/include/nlohmann/detail/macro_unscope.hpp index 8e1d49842..55b2ac99e 100644 --- a/include/nlohmann/detail/macro_unscope.hpp +++ b/include/nlohmann/detail/macro_unscope.hpp @@ -42,6 +42,7 @@ #undef JSON_BRACE_INIT_COPY_SEMANTICS #undef JSON_PRECISE_STREAM_POSITION #undef JSON_STRICT_NUL_HANDLING + #undef JSON_STRICT_BINARY_UTF8 #endif #include diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 9d844d85e..da5aa0cf4 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -115,6 +115,8 @@ class binary_writer /*! @param[in] j JSON value to serialize + @throw type_error.316 if JSON_STRICT_BINARY_UTF8 is enabled and a string + value or an object key is not valid UTF-8 @throw type_error.317 if @a j is not an object */ void write_bson(const BasicJsonType& j) @@ -145,6 +147,8 @@ class binary_writer /*! @param[in] j JSON value to serialize + @throw type_error.316 if JSON_STRICT_BINARY_UTF8 is enabled and a string + value or an object key is not valid UTF-8 */ void write_cbor(const BasicJsonType& j) { @@ -211,6 +215,8 @@ class binary_writer case value_t::string: { + check_text_utf8(*j.m_data.m_value.string, j); + // step 1: write control byte and the string length write_cbor_head(0x60, j.m_data.m_value.string->size()); @@ -287,6 +293,11 @@ class binary_writer // step 2: write each element for (const auto& el : *j.m_data.m_value.object) { + // el.first is checked here, against the object as + // diagnostics context, because write_cbor(el.first) + // converts it to a temporary basic_json that would be + // used as the context instead + check_text_utf8(el.first, j); write_cbor(el.first); write_cbor(el.second); } @@ -629,6 +640,8 @@ class binary_writer @param[in] add_prefix whether prefixes need to be used for this value @param[in] use_bjdata whether write in BJData format, default is false @param[in] bjdata_version which BJData version to use, default is draft2 + @throw type_error.316 if JSON_STRICT_BINARY_UTF8 is enabled and a string + value or an object key is not valid UTF-8 */ void write_ubjson(const BasicJsonType& j, const bool use_count, const bool use_type, const bool add_prefix = true, @@ -678,6 +691,8 @@ class binary_writer case value_t::string: { + check_text_utf8(*j.m_data.m_value.string, j); + if (add_prefix) { oa.write_character(to_char_type('S')); @@ -840,6 +855,7 @@ class binary_writer for (const auto& el : *j.m_data.m_value.object) { + check_text_utf8(el.first, j); write_number_with_ubjson_prefix(el.first.size(), true, use_bjdata); oa.write_characters( reinterpret_cast(el.first.data()), @@ -884,6 +900,10 @@ class binary_writer /*! @return The size of a BSON document entry header, including the id marker and the entry name size (and its null-terminator). + @throw out_of_range.409 if @a name contains U+0000, before anything is + written + @throw type_error.316 if JSON_STRICT_BINARY_UTF8 is enabled and @a name is + not valid UTF-8, before anything is written */ static std::size_t calc_bson_entry_header_size(const string_t& name, const BasicJsonType& j) { @@ -893,7 +913,8 @@ class binary_writer JSON_THROW(out_of_range::create(409, concat("BSON key cannot contain code point U+0000 (at byte ", std::to_string(it), ")"), &j)); } - static_cast(j); + check_text_utf8(name, j); + return /*id*/ 1ul + name.size() + /*zero-terminator*/1u; } @@ -949,9 +970,21 @@ class binary_writer /*! @return The size of the BSON-encoded string in @a value + @throw type_error.316 if JSON_STRICT_BINARY_UTF8 is enabled and @a value + is not valid UTF-8, before anything is written + + @note The UTF-8 check is skipped if @a value is already too long for the + 32-bit BSON length field (@ref to_bson_length rejects it later, once + the size of the whole document is known); this also keeps the check + from reading past a StringType that reports a size larger than what + it actually holds. */ - static std::size_t calc_bson_string_size(const string_t& value) + static std::size_t calc_bson_string_size(const string_t& value, const BasicJsonType& j) { + if (JSON_HEDLEY_LIKELY(value_in_range_of(value.size()))) + { + check_text_utf8(value, j); + } return sizeof(std::int32_t) + value.size() + 1ul; } @@ -1080,6 +1113,8 @@ class binary_writer is neither an object nor an array @throw out_of_range.415 if @a j is binary with a subtype that does not fit into a byte, before anything is written + @throw type_error.316 if JSON_STRICT_BINARY_UTF8 is enabled and @a j is a + string that is not valid UTF-8, before anything is written */ static std::size_t calc_bson_value_size(const BasicJsonType& j) { @@ -1101,7 +1136,7 @@ class binary_writer return calc_bson_unsigned_size(j.m_data.m_value.number_unsigned); case value_t::string: - return calc_bson_string_size(*j.m_data.m_value.string); + return calc_bson_string_size(*j.m_data.m_value.string, j); case value_t::null: return 0ul; @@ -1214,6 +1249,8 @@ class binary_writer written @throw out_of_range.415 if a binary value's subtype does not fit into a byte, before anything is written + @throw type_error.316 if JSON_STRICT_BINARY_UTF8 is enabled and a string + value or a key is not valid UTF-8, before anything is written */ static std::size_t calc_bson_sizes(const BasicJsonType& document, std::vector& nested_sizes) { @@ -2092,7 +2129,7 @@ class binary_writer */ void write_bon8_string(const string_t& s, bool& string_open, const BasicJsonType& context) { - check_bon8_utf8(s, context); + check_utf8(s, context); // a string that follows another string terminates it if (string_open) @@ -2122,7 +2159,7 @@ class binary_writer @throw type_error.316 if @a s is not valid UTF-8; the message names the first byte of the first invalid or incomplete sequence */ - static void check_bon8_utf8(const string_t& s, const BasicJsonType& context) + static void check_utf8(const string_t& s, const BasicJsonType& context) { static_cast(context); // only used when exceptions are enabled const auto* data = reinterpret_cast(s.data()); @@ -2133,6 +2170,29 @@ class binary_writer } } + /*! + @brief check a CBOR, UBJSON, BJData, or BSON text string for valid UTF-8 + + The check only happens if JSON_STRICT_BINARY_UTF8 is enabled. Otherwise, + the bytes are written unchanged, as before version 3.13.0. MessagePack + always writes the bytes as is, and BON8 always checks them (see + @ref check_utf8). + + @param[in] s the string to check + @param[in] context the value that holds @a s (for diagnostics) + @throw type_error.316 if JSON_STRICT_BINARY_UTF8 is enabled and @a s is + not valid UTF-8 + */ + static void check_text_utf8(const string_t& s, const BasicJsonType& context) + { +#if JSON_STRICT_BINARY_UTF8 + check_utf8(s, context); +#else + static_cast(s); + static_cast(context); +#endif + } + /*! @brief write an integer in the shortest encoding diff --git a/include/nlohmann/detail/string_utils.hpp b/include/nlohmann/detail/string_utils.hpp index 7c40f7395..2b6864d0d 100644 --- a/include/nlohmann/detail/string_utils.hpp +++ b/include/nlohmann/detail/string_utils.hpp @@ -117,13 +117,14 @@ This is a single-byte step of a "shift-based" UTF-8 decoder originally written by Björn Hoehrmann. See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details. -The library checks UTF-8 well-formedness (RFC 3629, section 4) in four +The library checks UTF-8 well-formedness (RFC 3629, section 4) in three places, which differ in speed, diagnostics, and how they read the input: -- decode() and @ref is_valid_utf8 below: the serializer (to escape and, in - strict mode, reject ill-formed UTF-8 when dumping a string) and the CBOR, - MessagePack, BSON, UBJSON and BJData readers (to reject ill-formed UTF-8 in - text strings at decode time). +- decode() below: the serializer, to escape and, in strict mode, reject + ill-formed UTF-8 when dumping a string. The CBOR, MessagePack, BSON, + UBJSON and BJData readers do not use it: none of those specs requires a + decoder to reject ill-formed UTF-8 in text strings, so the readers keep + the bytes as is and leave the check to dump() and the binary writers. - the per-lead-byte switch in lexer::scan_string(): JSON text, with a diagnostic for each kind of error. - validate_one_utf8() and valid_utf8_prefix() in string_scan.hpp: the lexer's @@ -178,38 +179,5 @@ inline std::uint8_t decode(std::uint8_t& state, std::uint32_t& codep, const std: return state; } -/*! -@brief check whether a string consists solely of valid UTF-8 - -Used by the CBOR/MessagePack/BSON/UBJSON binary readers to reject text -strings that are not valid UTF-8 at decode time (RFC 8949 §3.1 and the -MessagePack/BSON specifications all require text strings to be UTF-8), so -that malformed input is caught immediately instead of only surfacing later -as a type_error.316 when the resulting value is dumped. - -@param[in] s the string to check -@param[in] first index of the first byte to check; the bytes before it are - assumed to have been validated already and to end on a - code point boundary -@return whether @a s (from index @a first on) is valid UTF-8 -*/ -template -inline bool is_valid_utf8(const StringType& s, const std::size_t first = 0) noexcept -{ - std::uint8_t state = UTF8_ACCEPT; - std::uint32_t codepoint = 0; - - for (std::size_t i = first; i < s.size(); ++i) - { - decode(state, codepoint, static_cast(s[i])); - if (state == UTF8_REJECT) - { - return false; - } - } - - return state == UTF8_ACCEPT; -} - } // namespace detail NLOHMANN_JSON_NAMESPACE_END diff --git a/single_include/nlohmann/json_fwd.hpp b/single_include/nlohmann/json_fwd.hpp index 3ee7afa73..7cadc9b1c 100644 --- a/single_include/nlohmann/json_fwd.hpp +++ b/single_include/nlohmann/json_fwd.hpp @@ -63,6 +63,10 @@ #define JSON_STRICT_NUL_HANDLING 0 #endif +#ifndef JSON_STRICT_BINARY_UTF8 + #define JSON_STRICT_BINARY_UTF8 0 +#endif + #if JSON_DIAGNOSTICS #define NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS _diag #else @@ -99,14 +103,20 @@ #define NLOHMANN_JSON_ABI_TAG_STRICT_NUL_HANDLING #endif +#if JSON_STRICT_BINARY_UTF8 + #define NLOHMANN_JSON_ABI_TAG_STRICT_BINARY_UTF8 _sbu8 +#else + #define NLOHMANN_JSON_ABI_TAG_STRICT_BINARY_UTF8 +#endif + #ifndef NLOHMANN_JSON_NAMESPACE_NO_VERSION #define NLOHMANN_JSON_NAMESPACE_NO_VERSION 0 #endif // Construct the namespace ABI tags component -#define NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b, c, d, e, f) json_abi ## a ## b ## c ## d ## e ## f -#define NLOHMANN_JSON_ABI_TAGS_CONCAT(a, b, c, d, e, f) \ - NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b, c, d, e, f) +#define NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b, c, d, e, f, g) json_abi ## a ## b ## c ## d ## e ## f ## g +#define NLOHMANN_JSON_ABI_TAGS_CONCAT(a, b, c, d, e, f, g) \ + NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b, c, d, e, f, g) #define NLOHMANN_JSON_ABI_TAGS \ NLOHMANN_JSON_ABI_TAGS_CONCAT( \ @@ -115,7 +125,8 @@ NLOHMANN_JSON_ABI_TAG_DIAGNOSTIC_POSITIONS, \ NLOHMANN_JSON_ABI_TAG_BRACE_INIT_COPY_SEMANTICS, \ NLOHMANN_JSON_ABI_TAG_PRECISE_STREAM_POSITION, \ - NLOHMANN_JSON_ABI_TAG_STRICT_NUL_HANDLING) + NLOHMANN_JSON_ABI_TAG_STRICT_NUL_HANDLING, \ + NLOHMANN_JSON_ABI_TAG_STRICT_BINARY_UTF8) // Construct the namespace version component #define NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT_EX(major, minor, patch) \ diff --git a/tests/abi/config/default.cpp b/tests/abi/config/default.cpp index 879322dd0..e4c627060 100644 --- a/tests/abi/config/default.cpp +++ b/tests/abi/config/default.cpp @@ -44,6 +44,10 @@ TEST_CASE("default namespace") expected += "_snul"; #endif +#if JSON_STRICT_BINARY_UTF8 + expected += "_sbu8"; +#endif + expected += "_v" STRINGIZE(NLOHMANN_JSON_VERSION_MAJOR); expected += "_" STRINGIZE(NLOHMANN_JSON_VERSION_MINOR); expected += "_" STRINGIZE(NLOHMANN_JSON_VERSION_PATCH) "::basic_json"; diff --git a/tests/abi/config/noversion.cpp b/tests/abi/config/noversion.cpp index 4b1eb6ee4..858964695 100644 --- a/tests/abi/config/noversion.cpp +++ b/tests/abi/config/noversion.cpp @@ -45,6 +45,10 @@ TEST_CASE("default namespace without version component") expected += "_snul"; #endif +#if JSON_STRICT_BINARY_UTF8 + expected += "_sbu8"; +#endif + expected += "::basic_json"; // fallback for Clang diff --git a/tests/src/unit-binary_utf8_strict.cpp b/tests/src/unit-binary_utf8_strict.cpp new file mode 100644 index 000000000..c83b5938c --- /dev/null +++ b/tests/src/unit-binary_utf8_strict.cpp @@ -0,0 +1,110 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#include "doctest_compatibility.h" + +// The binary writers check strings and object keys for valid UTF-8 only if +// JSON_STRICT_BINARY_UTF8 is enabled (planned to be the default in 4.0.0). +// Without it, they write the bytes unchanged, as before version 3.13.0; the +// tests for that are next to the other tests of each format. +#ifdef JSON_STRICT_BINARY_UTF8 + #undef JSON_STRICT_BINARY_UTF8 +#endif + +#define JSON_STRICT_BINARY_UTF8 1 + +#include +using nlohmann::json; + +#include +#include + +TEST_CASE("JSON_STRICT_BINARY_UTF8 (see #5529, #5651)") +{ + SECTION("CBOR") + { + // a string value with ill-formed UTF-8 is rejected + CHECK_THROWS_WITH_AS(json::to_cbor(json("\xFF")), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xFF", json::type_error&); + // a truncated multi-byte sequence + CHECK_THROWS_WITH_AS(json::to_cbor(json("\xC3")), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xC3", json::type_error&); + // an encoded surrogate half (U+D800) + CHECK_THROWS_WITH_AS(json::to_cbor(json("\xED\xA0\x80")), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xED", json::type_error&); + // an overlong encoding of '.' + CHECK_THROWS_WITH_AS(json::to_cbor(json("\xC0\xAF")), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xC0", json::type_error&); + + // an object key with ill-formed UTF-8 is rejected the same way + CHECK_THROWS_WITH_AS(json::to_cbor(json{{"\xFF", 1}}), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xFF", json::type_error&); + + // binary values are not text and are unaffected + CHECK_NOTHROW(json::to_cbor(json::binary(std::vector({0xFF})))); + + // a value read back from CBOR with ill-formed bytes cannot be written + // back either (the reader is lenient regardless of the macro) + const json j = json::from_cbor(std::vector({0x62, 0xc0, 0xae})); + CHECK_THROWS_WITH_AS(json::to_cbor(j), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xC0", json::type_error&); + } + + SECTION("UBJSON") + { + CHECK_THROWS_WITH_AS(json::to_ubjson(json("\xFF")), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xFF", json::type_error&); + // a truncated multi-byte sequence + CHECK_THROWS_WITH_AS(json::to_ubjson(json("\xC3")), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xC3", json::type_error&); + // an encoded surrogate half (U+D800) + CHECK_THROWS_WITH_AS(json::to_ubjson(json("\xED\xA0\x80")), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xED", json::type_error&); + // an overlong encoding of '.' + CHECK_THROWS_WITH_AS(json::to_ubjson(json("\xC0\xAF")), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xC0", json::type_error&); + + // an object key with ill-formed UTF-8 is rejected the same way + CHECK_THROWS_WITH_AS(json::to_ubjson(json{{"\xFF", 1}}), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xFF", json::type_error&); + } + + SECTION("BJData") + { + CHECK_THROWS_WITH_AS(json::to_bjdata(json("\xFF")), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xFF", json::type_error&); + // a truncated multi-byte sequence + CHECK_THROWS_WITH_AS(json::to_bjdata(json("\xC3")), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xC3", json::type_error&); + // an encoded surrogate half (U+D800) + CHECK_THROWS_WITH_AS(json::to_bjdata(json("\xED\xA0\x80")), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xED", json::type_error&); + // an overlong encoding of '.' + CHECK_THROWS_WITH_AS(json::to_bjdata(json("\xC0\xAF")), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xC0", json::type_error&); + + // an object key with ill-formed UTF-8 is rejected the same way + CHECK_THROWS_WITH_AS(json::to_bjdata(json{{"\xFF", 1}}), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xFF", json::type_error&); + } + + SECTION("BSON") + { + // to_bson() rejects the same kind of ill-formed string value, before + // any bytes reach the output adapter (the BSON document length + // prefix must be known up front, so nothing is written incrementally) + std::vector out{0x42}; // a sentinel byte the writer must not touch + CHECK_THROWS_WITH_AS(json::to_bson(json{{"s", "\xFF"}}, nlohmann::detail::output_adapter(out)), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xFF", json::type_error&); + CHECK(out == std::vector {0x42}); + + CHECK_THROWS_WITH_AS(json::to_bson(json{{"s", "\xFF"}}), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xFF", json::type_error&); + // a truncated multi-byte sequence + CHECK_THROWS_WITH_AS(json::to_bson(json{{"s", "\xC3"}}), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xC3", json::type_error&); + // an encoded surrogate half (U+D800) + CHECK_THROWS_WITH_AS(json::to_bson(json{{"s", "\xED\xA0\x80"}}), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xED", json::type_error&); + // an overlong encoding of '.' + CHECK_THROWS_WITH_AS(json::to_bson(json{{"s", "\xC0\xAF"}}), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xC0", json::type_error&); + + // an object key with ill-formed UTF-8 is rejected as well; unlike + // the reader (which never validates element names), the writer + // checks both string values and object keys + CHECK_THROWS_WITH_AS(json::to_bson(json{{"\xFF", 1}}), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xFF", json::type_error&); + } + + SECTION("MessagePack and BON8 are unaffected") + { + // MessagePack allows any bytes in a str, so to_msgpack() writes them as + // is; BON8 always checks, because the lead bytes mark where strings end + CHECK(json::to_msgpack(json("\xFF")) == std::vector({0xa1, 0xff})); + CHECK_THROWS_AS(json::to_bon8(json("\xFF")), json::type_error&); + } +} diff --git a/tests/src/unit-bjdata.cpp b/tests/src/unit-bjdata.cpp index b6be66c9e..f34df1717 100644 --- a/tests/src/unit-bjdata.cpp +++ b/tests/src/unit-bjdata.cpp @@ -3906,6 +3906,43 @@ TEST_CASE("Universal Binary JSON Specification Examples 1") CHECK(json::to_bjdata(j) == v); CHECK(json::from_bjdata(v) == j); } + + SECTION("ill-formed UTF-8 (see #5529, #5651)") + { + // none of the binary format specs requires a decoder to reject + // ill-formed UTF-8 in a text string, so a value whose bytes are + // not valid UTF-8 (0xC0 0xAE is an overlong encoding of '.') + // round-trips byte for byte as a string value; to_bjdata() writes + // the bytes unchanged, as before 3.13.0, unless + // JSON_STRICT_BINARY_UTF8 is enabled (see + // unit-binary_utf8_strict.cpp) + const std::vector v = {'S', 'i', 2, 0xc0, 0xae}; + json j; + CHECK_NOTHROW(j = json::from_bjdata(v)); + REQUIRE(j.is_string()); + CHECK(j.get_ref() == std::string("\xc0\xae")); + CHECK_THROWS_AS(j.dump(), json::type_error&); + CHECK(json::from_bjdata(json::to_bjdata(j)) == j); + + // the same bytes as an object key round-trip as well + const std::vector v_key = {'{', 'i', 2, 0xc0, 0xae, 'i', 1, '}'}; + json j_key; + CHECK_NOTHROW(j_key = json::from_bjdata(v_key)); + REQUIRE(j_key.is_object()); + CHECK(j_key.contains(std::string("\xc0\xae"))); + CHECK(json::from_bjdata(json::to_bjdata(j_key)) == j_key); + + CHECK(json::from_bjdata(json::to_bjdata(json("\xFF"))) == json("\xFF")); + // a truncated multi-byte sequence + CHECK(json::from_bjdata(json::to_bjdata(json("\xC3"))) == json("\xC3")); + // an encoded surrogate half (U+D800) + CHECK(json::from_bjdata(json::to_bjdata(json("\xED\xA0\x80"))) == json("\xED\xA0\x80")); + // an overlong encoding of '.' + CHECK(json::from_bjdata(json::to_bjdata(json("\xC0\xAF"))) == json("\xC0\xAF")); + + // an object key with ill-formed UTF-8 is kept the same way + CHECK(json::from_bjdata(json::to_bjdata(json{{"\xFF", 1}})) == json{{"\xFF", 1}}); + } } SECTION("Array Type") diff --git a/tests/src/unit-bson.cpp b/tests/src/unit-bson.cpp index 2f9a727ab..92a14e6fe 100644 --- a/tests/src/unit-bson.cpp +++ b/tests/src/unit-bson.cpp @@ -154,6 +154,43 @@ TEST_CASE("BSON") #endif } + SECTION("ill-formed UTF-8 (see #5529, #5651)") + { + // a BSON document {"s": "\xC0\xAE"} (0xC0 0xAE is an overlong + // encoding of '.'); the BSON spec does not require a decoder to + // reject ill-formed UTF-8 in a string value, so the reader hands the + // bytes back unchanged + const std::vector v = + { + 0x0F, 0x00, 0x00, 0x00, // document length + 0x02, 's', 0x00, // type 0x02 (string), key "s" + 0x03, 0x00, 0x00, 0x00, // string length (including null) + 0xc0, 0xae, 0x00, // string content and its null terminator + 0x00 // document terminator + }; + json j; + CHECK_NOTHROW(j = json::from_bson(v)); + REQUIRE(j.is_object()); + REQUIRE(j.contains("s")); + CHECK(j["s"].get_ref() == std::string("\xc0\xae")); + // dump() still requires valid UTF-8 and throws for such a value + CHECK_THROWS_AS(j.dump(), json::type_error&); + // to_bson() writes the bytes back unchanged, as before 3.13.0, + // unless JSON_STRICT_BINARY_UTF8 is enabled (see unit-binary_utf8_strict.cpp) + CHECK(json::from_bson(json::to_bson(j)) == j); + + CHECK(json::from_bson(json::to_bson(json{{"s", "\xFF"}})) == json{{"s", "\xFF"}}); + // a truncated multi-byte sequence + CHECK(json::from_bson(json::to_bson(json{{"s", "\xC3"}})) == json{{"s", "\xC3"}}); + // an encoded surrogate half (U+D800) + CHECK(json::from_bson(json::to_bson(json{{"s", "\xED\xA0\x80"}})) == json{{"s", "\xED\xA0\x80"}}); + // an overlong encoding of '.' + CHECK(json::from_bson(json::to_bson(json{{"s", "\xC0\xAF"}})) == json{{"s", "\xC0\xAF"}}); + + // an object key with ill-formed UTF-8 is kept as well + CHECK(json::from_bson(json::to_bson(json{{"\xFF", 1}})) == json{{"\xFF", 1}}); + } + SECTION("lengths exceeding INT32_MAX cannot be serialized to BSON") { // out_of_range.412 is thrown from a single shared helper diff --git a/tests/src/unit-cbor.cpp b/tests/src/unit-cbor.cpp index c3a425a16..633f50fea 100644 --- a/tests/src/unit-cbor.cpp +++ b/tests/src/unit-cbor.cpp @@ -1801,19 +1801,41 @@ TEST_CASE("CBOR") CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector({0xA1, 0x7C, 0x01})), "[json.exception.parse_error.113] parse error at byte 2: syntax error while parsing CBOR string: expected length specification (0x60-0x7B) or indefinite string type (0x7F); last byte: 0x7C", json::parse_error&); } - SECTION("invalid UTF-8 in string (see #5529)") + SECTION("ill-formed UTF-8 in string (see #5529, #5651)") { + // RFC 8949 §3.1 leaves it up to the decoder whether to reject + // ill-formed UTF-8 in a text string; this library does not, and + // hands the original bytes back unchanged, matching the + // MessagePack reader and the behavior before #5185/#5531 (not in + // any release) + // a two-character text string (major type 3) whose bytes are not - // valid UTF-8 (0xC0 0xAE is an overlong encoding of '.') must be - // rejected at decode time, matching every other kind of - // malformed binary input, rather than only failing later when - // the resulting value is dumped - json _; - CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector({0x62, 0xc0, 0xae})), "[json.exception.parse_error.113] parse error at byte 3: syntax error while parsing CBOR string: invalid string: ill-formed UTF-8 byte", json::parse_error&); - CHECK(json::from_cbor(std::vector({0x62, 0xc0, 0xae}), true, false).is_discarded()); + // valid UTF-8 (0xC0 0xAE is an overlong encoding of '.') round-trips + // byte for byte as a string value + const std::vector ill_formed_value = {0x62, 0xc0, 0xae}; + json j_value; + CHECK_NOTHROW(j_value = json::from_cbor(ill_formed_value)); + REQUIRE(j_value.is_string()); + CHECK(j_value.get_ref() == std::string("\xc0\xae")); + // dump() still requires valid UTF-8 and throws for such a value, + // unless an error handler that replaces or ignores the bytes is + // passed + CHECK_THROWS_AS(j_value.dump(), json::type_error&); + // to_cbor() writes the bytes back unchanged, as before 3.13.0, + // unless JSON_STRICT_BINARY_UTF8 is enabled (see unit-binary_utf8_strict.cpp) + CHECK(json::from_cbor(json::to_cbor(j_value)) == j_value); + + // the same bytes as an object key round-trip as well + const std::vector ill_formed_key = {0xa1, 0x62, 0xc0, 0xae, 0x01}; + json j_key; + CHECK_NOTHROW(j_key = json::from_cbor(ill_formed_key)); + REQUIRE(j_key.is_object()); + CHECK(j_key.contains(std::string("\xc0\xae"))); + CHECK(json::from_cbor(json::to_cbor(j_key)) == j_key); // a CBOR byte string (major type 2) with the very same bytes is // NOT text and must still be accepted as-is + json _; CHECK_NOTHROW(_ = json::from_cbor(std::vector({0x42, 0xc0, 0xae}))); CHECK(_ == json::binary(std::vector({0xc0, 0xae}))); @@ -1822,17 +1844,47 @@ TEST_CASE("CBOR") CHECK(json::from_cbor(json::to_cbor(j)) == j); } - SECTION("invalid UTF-8 in indefinite-length string") + SECTION("to_cbor keeps ill-formed UTF-8 (see #5651)") + { + // to_cbor() writes the bytes unchanged, as before 3.13.0, unless + // JSON_STRICT_BINARY_UTF8 is enabled (see + // unit-binary_utf8_strict.cpp); from_cbor() reads them back as is + CHECK(json::from_cbor(json::to_cbor(json("\xFF"))) == json("\xFF")); + // a truncated multi-byte sequence + CHECK(json::from_cbor(json::to_cbor(json("\xC3"))) == json("\xC3")); + // an encoded surrogate half (U+D800) + CHECK(json::from_cbor(json::to_cbor(json("\xED\xA0\x80"))) == json("\xED\xA0\x80")); + // an overlong encoding of '.' + CHECK(json::from_cbor(json::to_cbor(json("\xC0\xAF"))) == json("\xC0\xAF")); + + // an object key with ill-formed UTF-8 is kept the same way + CHECK(json::from_cbor(json::to_cbor(json{{"\xFF", 1}})) == json{{"\xFF", 1}}); + + // binary values are not text and are unaffected + CHECK_NOTHROW(json::to_cbor(json::binary(std::vector({0xFF})))); + } + + SECTION("ill-formed UTF-8 in indefinite-length string") { json _; - // every chunk must be valid UTF-8 on its own (RFC 8949, Section - // 3.2.3), so a code point split across two chunks is rejected - CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector({0x7f, 0x61, 0xc3, 0x61, 0xa9, 0xff})), "[json.exception.parse_error.113] parse error at byte 3: syntax error while parsing CBOR string: invalid string: ill-formed UTF-8 byte", json::parse_error&); - CHECK(json::from_cbor(std::vector({0x7f, 0x61, 0xc3, 0x61, 0xa9, 0xff}), true, false).is_discarded()); + // the chunks are concatenated as is, without checking that each + // chunk is valid UTF-8 on its own (RFC 8949, Section 3.2.3), so + // a code point split across two chunks yields a valid string + CHECK_NOTHROW(_ = json::from_cbor(std::vector({0x7f, 0x61, 0xc3, 0x61, 0xa9, 0xff}))); + CHECK(_ == "\xc3\xa9"); + CHECK(_.dump() == "\"\xc3\xa9\""); - // an ill-formed later chunk is rejected after valid ones - CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector({0x7f, 0x62, 0xc3, 0xa9, 0x62, 0xc0, 0xae, 0xff})), "[json.exception.parse_error.113] parse error at byte 7: syntax error while parsing CBOR string: invalid string: ill-formed UTF-8 byte", json::parse_error&); + // a truncated code point is kept as is + CHECK_NOTHROW(_ = json::from_cbor(std::vector({0x7f, 0x61, 0xc3, 0xff}))); + CHECK(_ == "\xc3"); + CHECK_THROWS_AS(_.dump(), json::type_error&); + CHECK(json::from_cbor(json::to_cbor(_)) == _); + + // an ill-formed later chunk is kept after valid ones + CHECK_NOTHROW(_ = json::from_cbor(std::vector({0x7f, 0x62, 0xc3, 0xa9, 0x62, 0xc0, 0xae, 0xff}))); + CHECK(_ == "\xc3\xa9\xc0\xae"); + CHECK_THROWS_AS(_.dump(), json::type_error&); // valid multi-byte chunks are accepted CHECK(json::from_cbor(std::vector({0x7f, 0x62, 0xc3, 0xa9, 0x62, 0xc3, 0xb6, 0xff})) == "\xc3\xa9\xc3\xb6"); @@ -1840,9 +1892,6 @@ TEST_CASE("CBOR") SECTION("many chunks in indefinite-length string") { - // only the newly read chunk is validated, not the whole string - // collected so far; validating the latter made this input take - // quadratic time (about ten seconds for 100000 chunks) constexpr std::size_t chunks = 100000; std::vector v{0x7f}; for (std::size_t i = 0; i < chunks; ++i) diff --git a/tests/src/unit-msgpack.cpp b/tests/src/unit-msgpack.cpp index afabb85c6..a92496144 100644 --- a/tests/src/unit-msgpack.cpp +++ b/tests/src/unit-msgpack.cpp @@ -1540,19 +1540,39 @@ TEST_CASE("MessagePack") CHECK_THROWS_WITH_AS(_ = json::from_msgpack(std::vector({0x81})), "[json.exception.parse_error.110] parse error at byte 2: syntax error while parsing MessagePack string: unexpected end of input", json::parse_error&); } - SECTION("invalid UTF-8 in string (see #5529)") + SECTION("ill-formed UTF-8 in string (see #5529, #5651)") { + // the MessagePack specification explicitly allows a str object to + // contain a byte sequence that is not valid UTF-8 and expects a + // deserializer to hand the original bytes back unchanged; this + // library follows that, unlike CBOR/UBJSON/BJData/BSON, whose + // specifications require text strings to be valid UTF-8 + // a fixstr of length 2 (0xA0 | 2) whose bytes are not valid UTF-8 - // (0xC0 0xAE is an overlong encoding of '.') must be rejected at - // decode time, matching every other kind of malformed binary - // input, rather than only failing later when the resulting - // value is dumped - json _; - CHECK_THROWS_WITH_AS(_ = json::from_msgpack(std::vector({0xa2, 0xc0, 0xae})), "[json.exception.parse_error.113] parse error at byte 3: syntax error while parsing MessagePack string: invalid string: ill-formed UTF-8 byte", json::parse_error&); - CHECK(json::from_msgpack(std::vector({0xa2, 0xc0, 0xae}), true, false).is_discarded()); + // (0xC0 0xAE is an overlong encoding of '.') round-trips byte for + // byte as a string value + const std::vector ill_formed_value = {0xa2, 0xc0, 0xae}; + json j_value; + CHECK_NOTHROW(j_value = json::from_msgpack(ill_formed_value)); + REQUIRE(j_value.is_string()); + CHECK(j_value.get_ref() == std::string("\xc0\xae")); + CHECK(json::from_msgpack(json::to_msgpack(j_value)) == j_value); + // dump() still requires valid UTF-8 and throws for such a value, + // unless an error handler that replaces or ignores the bytes is + // passed + CHECK_THROWS_AS(j_value.dump(), json::type_error&); + + // the same bytes as an object key round-trip as well + const std::vector ill_formed_key = {0x81, 0xa2, 0xc0, 0xae, 0x01}; + json j_key; + CHECK_NOTHROW(j_key = json::from_msgpack(ill_formed_key)); + REQUIRE(j_key.is_object()); + CHECK(j_key.contains(std::string("\xc0\xae"))); + CHECK(json::from_msgpack(json::to_msgpack(j_key)) == j_key); // a MessagePack bin8 blob with the very same bytes is NOT text // and must still be accepted as-is + json _; CHECK_NOTHROW(_ = json::from_msgpack(std::vector({0xc4, 0x02, 0xc0, 0xae}))); CHECK(_ == json::binary(std::vector({0xc0, 0xae}))); diff --git a/tests/src/unit-ubjson.cpp b/tests/src/unit-ubjson.cpp index 9a9710b48..450882a12 100644 --- a/tests/src/unit-ubjson.cpp +++ b/tests/src/unit-ubjson.cpp @@ -2505,6 +2505,43 @@ TEST_CASE("Universal Binary JSON Specification Examples 1") CHECK(json::to_ubjson(j) == v); CHECK(json::from_ubjson(v) == j); } + + SECTION("ill-formed UTF-8 (see #5529, #5651)") + { + // none of the binary format specs requires a decoder to reject + // ill-formed UTF-8 in a text string, so a value whose bytes are + // not valid UTF-8 (0xC0 0xAE is an overlong encoding of '.') + // round-trips byte for byte as a string value; to_ubjson() writes + // the bytes unchanged, as before 3.13.0, unless + // JSON_STRICT_BINARY_UTF8 is enabled (see + // unit-binary_utf8_strict.cpp) + const std::vector v = {'S', 'i', 2, 0xc0, 0xae}; + json j; + CHECK_NOTHROW(j = json::from_ubjson(v)); + REQUIRE(j.is_string()); + CHECK(j.get_ref() == std::string("\xc0\xae")); + CHECK_THROWS_AS(j.dump(), json::type_error&); + CHECK(json::from_ubjson(json::to_ubjson(j)) == j); + + // the same bytes as an object key round-trip as well + const std::vector v_key = {'{', 'i', 2, 0xc0, 0xae, 'i', 1, '}'}; + json j_key; + CHECK_NOTHROW(j_key = json::from_ubjson(v_key)); + REQUIRE(j_key.is_object()); + CHECK(j_key.contains(std::string("\xc0\xae"))); + CHECK(json::from_ubjson(json::to_ubjson(j_key)) == j_key); + + CHECK(json::from_ubjson(json::to_ubjson(json("\xFF"))) == json("\xFF")); + // a truncated multi-byte sequence + CHECK(json::from_ubjson(json::to_ubjson(json("\xC3"))) == json("\xC3")); + // an encoded surrogate half (U+D800) + CHECK(json::from_ubjson(json::to_ubjson(json("\xED\xA0\x80"))) == json("\xED\xA0\x80")); + // an overlong encoding of '.' + CHECK(json::from_ubjson(json::to_ubjson(json("\xC0\xAF"))) == json("\xC0\xAF")); + + // an object key with ill-formed UTF-8 is kept the same way + CHECK(json::from_ubjson(json::to_ubjson(json{{"\xFF", 1}})) == json{{"\xFF", 1}}); + } } SECTION("Array Type") From 73e9eae3c135e262dac3fe3e7978b46694e531d3 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 4 Oct 2026 12:13:49 +0200 Subject: [PATCH 20/22] Add an error_handler parameter for UTF-8 to the binary readers and writers (#5746) Signed-off-by: Niels Lohmann --- docs/mkdocs/docs/api/basic_json/dump.md | 9 +- .../docs/api/basic_json/error_handler_t.md | 32 +- .../mkdocs/docs/api/basic_json/from_bjdata.md | 15 +- docs/mkdocs/docs/api/basic_json/from_bson.md | 15 +- docs/mkdocs/docs/api/basic_json/from_cbor.md | 18 +- .../docs/api/basic_json/from_msgpack.md | 18 +- .../mkdocs/docs/api/basic_json/from_ubjson.md | 15 +- docs/mkdocs/docs/api/basic_json/to_bjdata.md | 26 +- docs/mkdocs/docs/api/basic_json/to_bson.md | 28 +- docs/mkdocs/docs/api/basic_json/to_cbor.md | 26 +- docs/mkdocs/docs/api/basic_json/to_msgpack.md | 20 +- docs/mkdocs/docs/api/basic_json/to_ubjson.md | 26 +- .../api/macros/json_strict_binary_utf8.md | 18 +- docs/mkdocs/docs/examples/error_handler_t.cpp | 3 +- .../docs/examples/error_handler_t.output | 1 + .../docs/features/binary_formats/bjdata.md | 24 +- .../docs/features/binary_formats/bson.md | 19 +- .../docs/features/binary_formats/cbor.md | 17 +- .../features/binary_formats/messagepack.md | 18 +- .../docs/features/binary_formats/ubjson.md | 24 +- docs/mkdocs/docs/home/exceptions.md | 18 +- .../nlohmann/detail/input/binary_reader.hpp | 114 ++- .../nlohmann/detail/output/binary_writer.hpp | 213 +++-- .../nlohmann/detail/output/error_handler.hpp | 50 ++ include/nlohmann/detail/output/serializer.hpp | 72 +- include/nlohmann/detail/string_utils.hpp | 130 ++++ include/nlohmann/json.hpp | 135 ++-- single_include/nlohmann/json.hpp | 734 ++++++++++++++---- tests/src/unit-binary_utf8_error_handler.cpp | 362 +++++++++ tests/src/unit-binary_utf8_strict.cpp | 16 +- 30 files changed, 1794 insertions(+), 422 deletions(-) create mode 100644 include/nlohmann/detail/output/error_handler.hpp create mode 100644 tests/src/unit-binary_utf8_error_handler.cpp diff --git a/docs/mkdocs/docs/api/basic_json/dump.md b/docs/mkdocs/docs/api/basic_json/dump.md index a3c9db3a8..d805f3db1 100644 --- a/docs/mkdocs/docs/api/basic_json/dump.md +++ b/docs/mkdocs/docs/api/basic_json/dump.md @@ -25,10 +25,12 @@ and `ensure_ascii` parameters. result consists of ASCII characters only. `error_handler` (in) -: how to react on decoding errors; there are three possible values (see [`error_handler_t`](error_handler_t.md): +: how to react on decoding errors; there are four possible values (see [`error_handler_t`](error_handler_t.md): `strict` (throws an exception in case a decoding error occurs; default), `replace` (replace invalid UTF-8 sequences - with U+FFFD), and `ignore` (ignore invalid UTF-8 sequences during serialization; all valid bytes are copied to the - output unchanged, and invalid bytes are dropped)). + with U+FFFD), `ignore` (ignore invalid UTF-8 sequences during serialization; all valid bytes are copied to the + output unchanged, and invalid bytes are dropped), and `keep` (write the ill-formed bytes to the output as is, + without escaping them, even if `ensure_ascii` is `#!cpp true`; the result is then not valid UTF-8, but equals the + input bytes exactly, and well-formed characters around the ill-formed bytes are still escaped as usual)). ## Return value @@ -94,3 +96,4 @@ Binary values are serialized as an object containing two keys: - Indentation character `indent_char`, option `ensure_ascii` and exceptions added in version 3.0.0. - Error handlers added in version 3.4.0. - Serialization of binary values added in version 3.8.0. +- Error handler `keep` added in version 3.13.0. diff --git a/docs/mkdocs/docs/api/basic_json/error_handler_t.md b/docs/mkdocs/docs/api/basic_json/error_handler_t.md index 51dc6510f..17327fe26 100644 --- a/docs/mkdocs/docs/api/basic_json/error_handler_t.md +++ b/docs/mkdocs/docs/api/basic_json/error_handler_t.md @@ -4,15 +4,31 @@ enum class error_handler_t { strict, replace, - ignore + ignore, + keep }; ``` -This enumeration is used in the [`dump`](dump.md) function to choose how to treat decoding errors while serializing a -`basic_json` value. Three values are differentiated: +This enumeration is used to choose how to treat ill-formed UTF-8 in a string value or object key: + +- [`dump`](dump.md) uses it while serializing a `basic_json` value to text. +- [`to_cbor`](to_cbor.md), [`to_msgpack`](to_msgpack.md), [`to_ubjson`](to_ubjson.md), [`to_bjdata`](to_bjdata.md), + and [`to_bson`](to_bson.md) use it while serializing a `basic_json` value to that binary format. Their default is + `keep`, as no binary writer checked before this parameter was added. CBOR, UBJSON, BJData, and BSON require valid + UTF-8, so for these four the default is `strict` if [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) + is enabled; MessagePack's specification explicitly allows a string to contain ill-formed UTF-8, so `to_msgpack` + stays at `keep`. `to_bon8` does not take this parameter: BON8 always validates, since UTF-8 lead bytes are + structural to that format. +- [`from_cbor`](from_cbor.md), [`from_msgpack`](from_msgpack.md), [`from_ubjson`](from_ubjson.md), + [`from_bjdata`](from_bjdata.md), and [`from_bson`](from_bson.md) use it while parsing that binary format, to decide + whether to check a string value or object key for well-formed UTF-8 at all; by default (`keep`) they do not, as no + binary reader did before this parameter was added. `from_bon8` does not take this parameter, for the same reason + `to_bon8` does not. + +Four values are differentiated: strict -: throw a `type_error` exception in case of invalid UTF-8 +: throw a `type_error`/`parse_error` exception in case of invalid UTF-8 replace : replace invalid UTF-8 sequences with U+FFFD (� REPLACEMENT CHARACTER) @@ -20,6 +36,12 @@ replace ignore : ignore invalid UTF-8 sequences; all valid bytes are copied to the output unchanged, and invalid bytes are dropped +keep +: keep invalid UTF-8 sequences unchanged; only meaningful for the binary formats mentioned above, since [`dump`] + (dump.md) itself must produce text, and `keep` there writes the ill-formed bytes to the output as is, so the + result is then not valid UTF-8 (but still equals the input bytes exactly, including around any well-formed + characters, which are still escaped as usual) + ## Examples ??? example @@ -45,3 +67,5 @@ ignore ## Version history - Added in version 3.4.0. +- Added `keep`, and made this enumeration apply to the binary readers and writers in addition to `dump`, in version + 3.13.0. diff --git a/docs/mkdocs/docs/api/basic_json/from_bjdata.md b/docs/mkdocs/docs/api/basic_json/from_bjdata.md index 9df21ab30..a45e00ad5 100644 --- a/docs/mkdocs/docs/api/basic_json/from_bjdata.md +++ b/docs/mkdocs/docs/api/basic_json/from_bjdata.md @@ -5,12 +5,14 @@ template static basic_json from_bjdata(InputType&& i, const bool strict = true, - const bool allow_exceptions = true); + const bool allow_exceptions = true, + const error_handler_t error_handler = error_handler_t::keep); // (2) template static basic_json from_bjdata(IteratorType first, SentinelType last, const bool strict = true, - const bool allow_exceptions = true); + const bool allow_exceptions = true, + const error_handler_t error_handler = error_handler_t::keep); ``` Deserializes a given input to a JSON value using the BJData (Binary JData) serialization format. @@ -58,6 +60,12 @@ The exact mapping and its limitations are described on a [dedicated page](../../ `allow_exceptions` (in) : whether to throw exceptions in case of a parse error (optional, `#!cpp true` by default) +`error_handler` (in) +: how to treat a string value or object key that is not valid UTF-8; see [`error_handler_t`](error_handler_t.md). + BJData does not require a decoder to reject ill-formed UTF-8, so checking is opt-in: the default, `keep`, does not + check at all, as every binary reader did before this parameter was added; `strict` checks and throws; + `replace`/`ignore` sanitize the string the same way [`dump`](dump.md) would + ## Return value deserialized JSON value; in case of a parse error and `allow_exceptions` set to `#!cpp false`, the return value will be @@ -73,7 +81,7 @@ Strong guarantee: if an exception is thrown, there are no changes in the JSON va the end of the file was not reached when `strict` was set to true - Throws [parse_error.112](../../home/exceptions.md#jsonexceptionparse_error112) if a parse error occurs - Throws [parse_error.113](../../home/exceptions.md#jsonexceptionparse_error113) if a string could not be parsed - successfully + successfully, or if a string value or object key is not valid UTF-8 and `error_handler` is `strict` - Throws [out_of_range.408](../../home/exceptions.md#jsonexceptionout_of_range408) if the size of an optimized container or n-dimensional array cannot be represented by `std::size_t` @@ -111,3 +119,4 @@ Linear in the size of the input. - Added in version 3.11.0. - Extended container support (1) to include types with lvalue-only ADL `begin`/`end` (matching `std::begin`/`std::end` semantics) in version 3.13.0. - Extended overload (2) to accept heterogeneous iterator+sentinel pairs (C++20 ranges support) in version 3.13.0. +- Added `error_handler` parameter in version 3.13.0. diff --git a/docs/mkdocs/docs/api/basic_json/from_bson.md b/docs/mkdocs/docs/api/basic_json/from_bson.md index 9dfd9dc18..b037e8e07 100644 --- a/docs/mkdocs/docs/api/basic_json/from_bson.md +++ b/docs/mkdocs/docs/api/basic_json/from_bson.md @@ -5,12 +5,14 @@ template static basic_json from_bson(InputType&& i, const bool strict = true, - const bool allow_exceptions = true); + const bool allow_exceptions = true, + const error_handler_t error_handler = error_handler_t::keep); // (2) template static basic_json from_bson(IteratorType first, SentinelType last, const bool strict = true, - const bool allow_exceptions = true); + const bool allow_exceptions = true, + const error_handler_t error_handler = error_handler_t::keep); ``` Deserializes a given input to a JSON value using the BSON (Binary JSON) serialization format. @@ -58,6 +60,12 @@ The exact mapping and its limitations are described on a [dedicated page](../../ `allow_exceptions` (in) : whether to throw exceptions in case of a parse error (optional, `#!cpp true` by default) +`error_handler` (in) +: how to treat a string value or object key that is not valid UTF-8; see [`error_handler_t`](error_handler_t.md). + BSON does not require a decoder to reject ill-formed UTF-8, so checking is opt-in: the default, `keep`, does not + check at all, as every binary reader did before this parameter was added; `strict` checks and throws; + `replace`/`ignore` sanitize the string the same way [`dump`](dump.md) would + ## Return value deserialized JSON value; in case of a parse error and `allow_exceptions` set to `#!cpp false`, the return value will be @@ -75,6 +83,8 @@ Strong guarantee: if an exception is thrown, there are no changes in the JSON va invalid string or byte array length) - Throws [`parse_error.114`](../../home/exceptions.md#jsonexceptionparse_error114) if an unsupported BSON record type is encountered +- Throws [`parse_error.113`](../../home/exceptions.md#jsonexceptionparse_error113) if a string value or object key is + not valid UTF-8 and `error_handler` is `strict` ## Complexity @@ -111,6 +121,7 @@ Linear in the size of the input. - Added in version 3.4.0. - Extended container support (1) to include types with lvalue-only ADL `begin`/`end` (matching `std::begin`/`std::end` semantics) in version 3.13.0. - Extended overload (2) to accept heterogeneous iterator+sentinel pairs (C++20 ranges support) in version 3.13.0. +- Added `error_handler` parameter in version 3.13.0. !!! warning "Deprecation" diff --git a/docs/mkdocs/docs/api/basic_json/from_cbor.md b/docs/mkdocs/docs/api/basic_json/from_cbor.md index 791c183bb..7d7df08a5 100644 --- a/docs/mkdocs/docs/api/basic_json/from_cbor.md +++ b/docs/mkdocs/docs/api/basic_json/from_cbor.md @@ -6,14 +6,16 @@ template static basic_json from_cbor(InputType&& i, const bool strict = true, const bool allow_exceptions = true, - const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error); + const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error, + const error_handler_t error_handler = error_handler_t::keep); // (2) template static basic_json from_cbor(IteratorType first, SentinelType last, const bool strict = true, const bool allow_exceptions = true, - const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error); + const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error, + const error_handler_t error_handler = error_handler_t::keep); ``` Deserializes a given input to a JSON value using the CBOR (Concise Binary Object Representation) serialization format. @@ -65,6 +67,12 @@ The exact mapping and its limitations are described on a [dedicated page](../../ : how to treat CBOR tags (optional, `error` by default); see [`cbor_tag_handler_t`](cbor_tag_handler_t.md) for more information +`error_handler` (in) +: how to treat a string value or object key that is not valid UTF-8; see [`error_handler_t`](error_handler_t.md). + CBOR does not require a decoder to reject ill-formed UTF-8, so checking is opt-in: the default, `keep`, does not + check at all, as every binary reader did before this parameter was added; `strict` checks and throws; + `replace`/`ignore` sanitize the string the same way [`dump`](dump.md) would + ## Return value deserialized JSON value; in case of a parse error and `allow_exceptions` set to `#!cpp false`, the return value will be @@ -80,8 +88,9 @@ Strong guarantee: if an exception is thrown, there are no changes in the JSON va the end of the file was not reached when `strict` was set to true - Throws [parse_error.112](../../home/exceptions.md#jsonexceptionparse_error112) if unsupported features from CBOR were used in the given input or if the input is not valid CBOR -- Throws [parse_error.113](../../home/exceptions.md#jsonexceptionparse_error113) if a map key is not a string (keys of other - types are not supported, as JSON object keys are always strings) or a string is malformed +- Throws [parse_error.113](../../home/exceptions.md#jsonexceptionparse_error113) if a map key is not a string (keys of + other types are not supported, as JSON object keys are always strings), or if a string value or object key is not + valid UTF-8 and `error_handler` is `strict` ## Complexity @@ -121,6 +130,7 @@ Linear in the size of the input. - Added `tag_handler` parameter in version 3.9.0. - Extended container support (1) to include types with lvalue-only ADL `begin`/`end` (matching `std::begin`/`std::end` semantics) in version 3.13.0. - Extended overload (2) to accept heterogeneous iterator+sentinel pairs (C++20 ranges support) in version 3.13.0. +- Added `error_handler` parameter in version 3.13.0. !!! warning "Deprecation" diff --git a/docs/mkdocs/docs/api/basic_json/from_msgpack.md b/docs/mkdocs/docs/api/basic_json/from_msgpack.md index 395512acb..2e7fa5062 100644 --- a/docs/mkdocs/docs/api/basic_json/from_msgpack.md +++ b/docs/mkdocs/docs/api/basic_json/from_msgpack.md @@ -5,12 +5,14 @@ template static basic_json from_msgpack(InputType&& i, const bool strict = true, - const bool allow_exceptions = true); + const bool allow_exceptions = true, + const error_handler_t error_handler = error_handler_t::keep); // (2) template static basic_json from_msgpack(IteratorType first, SentinelType last, const bool strict = true, - const bool allow_exceptions = true); + const bool allow_exceptions = true, + const error_handler_t error_handler = error_handler_t::keep); ``` Deserializes a given input to a JSON value using the MessagePack serialization format. @@ -58,6 +60,12 @@ The exact mapping and its limitations are described on a [dedicated page](../../ `allow_exceptions` (in) : whether to throw exceptions in case of a parse error (optional, `#!cpp true` by default) +`error_handler` (in) +: how to treat a string value or object key that is not valid UTF-8; see [`error_handler_t`](error_handler_t.md). + MessagePack's specification explicitly allows ill-formed UTF-8, so checking is opt-in: the default, `keep`, does + not check at all, as every binary reader did before this parameter was added; `strict` checks and throws; + `replace`/`ignore` sanitize the string the same way [`dump`](dump.md) would + ## Return value deserialized JSON value; in case of a parse error and `allow_exceptions` set to `#!cpp false`, the return value will be @@ -73,8 +81,9 @@ Strong guarantee: if an exception is thrown, there are no changes in the JSON va the end of the file was not reached when `strict` was set to true - Throws [parse_error.112](../../home/exceptions.md#jsonexceptionparse_error112) if unsupported features from MessagePack were used in the given input or if the input is not valid MessagePack -- Throws [parse_error.113](../../home/exceptions.md#jsonexceptionparse_error113) if a map key is not a string (keys of other - types are not supported, as JSON object keys are always strings) or a string is malformed +- Throws [parse_error.113](../../home/exceptions.md#jsonexceptionparse_error113) if a map key is not a string (keys of + other types are not supported, as JSON object keys are always strings), or if a string value or object key is not + valid UTF-8 and `error_handler` is `strict` ## Complexity @@ -113,6 +122,7 @@ Linear in the size of the input. - Added `allow_exceptions` parameter in version 3.2.0. - Extended container support (1) to include types with lvalue-only ADL `begin`/`end` (matching `std::begin`/`std::end` semantics) in version 3.13.0. - Extended overload (2) to accept heterogeneous iterator+sentinel pairs (C++20 ranges support) in version 3.13.0. +- Added `error_handler` parameter in version 3.13.0. !!! warning "Deprecation" diff --git a/docs/mkdocs/docs/api/basic_json/from_ubjson.md b/docs/mkdocs/docs/api/basic_json/from_ubjson.md index 1ad076588..ac9404d4c 100644 --- a/docs/mkdocs/docs/api/basic_json/from_ubjson.md +++ b/docs/mkdocs/docs/api/basic_json/from_ubjson.md @@ -5,12 +5,14 @@ template static basic_json from_ubjson(InputType&& i, const bool strict = true, - const bool allow_exceptions = true); + const bool allow_exceptions = true, + const error_handler_t error_handler = error_handler_t::keep); // (2) template static basic_json from_ubjson(IteratorType first, SentinelType last, const bool strict = true, - const bool allow_exceptions = true); + const bool allow_exceptions = true, + const error_handler_t error_handler = error_handler_t::keep); ``` Deserializes a given input to a JSON value using the UBJSON (Universal Binary JSON) serialization format. @@ -58,6 +60,12 @@ The exact mapping and its limitations are described on a [dedicated page](../../ `allow_exceptions` (in) : whether to throw exceptions in case of a parse error (optional, `#!cpp true` by default) +`error_handler` (in) +: how to treat a string value or object key that is not valid UTF-8; see [`error_handler_t`](error_handler_t.md). + UBJSON does not require a decoder to reject ill-formed UTF-8, so checking is opt-in: the default, `keep`, does not + check at all, as every binary reader did before this parameter was added; `strict` checks and throws; + `replace`/`ignore` sanitize the string the same way [`dump`](dump.md) would + ## Return value deserialized JSON value; in case of a parse error and `allow_exceptions` set to `#!cpp false`, the return value will be @@ -73,7 +81,7 @@ Strong guarantee: if an exception is thrown, there are no changes in the JSON va the end of the file was not reached when `strict` was set to true - Throws [parse_error.112](../../home/exceptions.md#jsonexceptionparse_error112) if a parse error occurs - Throws [parse_error.113](../../home/exceptions.md#jsonexceptionparse_error113) if a string could not be parsed - successfully + successfully, or if a string value or object key is not valid UTF-8 and `error_handler` is `strict` - Throws [out_of_range.408](../../home/exceptions.md#jsonexceptionout_of_range408) if the size of an optimized container or n-dimensional array cannot be represented by `std::size_t` @@ -112,6 +120,7 @@ Linear in the size of the input. - Added `allow_exceptions` parameter in version 3.2.0. - Extended container support (1) to include types with lvalue-only ADL `begin`/`end` (matching `std::begin`/`std::end` semantics) in version 3.13.0. - Extended overload (2) to accept heterogeneous iterator+sentinel pairs (C++20 ranges support) in version 3.13.0. +- Added `error_handler` parameter in version 3.13.0. !!! warning "Deprecation" diff --git a/docs/mkdocs/docs/api/basic_json/to_bjdata.md b/docs/mkdocs/docs/api/basic_json/to_bjdata.md index 63dc5379e..df3b69004 100644 --- a/docs/mkdocs/docs/api/basic_json/to_bjdata.md +++ b/docs/mkdocs/docs/api/basic_json/to_bjdata.md @@ -5,15 +5,18 @@ static std::vector to_bjdata(const basic_json& j, const bool use_size = false, const bool use_type = false, - const bjdata_version_t version = bjdata_version_t::draft2); + const bjdata_version_t version = bjdata_version_t::draft2, + const error_handler_t error_handler = error_handler_t::keep); // (2) static void to_bjdata(const basic_json& j, detail::output_adapter o, const bool use_size = false, const bool use_type = false, - const bjdata_version_t version = bjdata_version_t::draft2); + const bjdata_version_t version = bjdata_version_t::draft2, + const error_handler_t error_handler = error_handler_t::keep); static void to_bjdata(const basic_json& j, detail::output_adapter o, const bool use_size = false, const bool use_type = false, - const bjdata_version_t version = bjdata_version_t::draft2); + const bjdata_version_t version = bjdata_version_t::draft2, + const error_handler_t error_handler = error_handler_t::keep); ``` Serializes a given JSON value `j` to a byte vector using the BJData (Binary JData) serialization format. BJData aims to @@ -43,6 +46,12 @@ The exact mapping and its limitations are described on a [dedicated page](../../ : which version of BJData to use (see note on "Binary values" on [BJData](../../features/binary_formats/bjdata.md)); optional, `#!cpp bjdata_version_t::draft2` by default. +`error_handler` (in) +: how to treat a string or object key in `j` that is not valid UTF-8; see [`error_handler_t`](error_handler_t.md). + The default, `keep`, writes the ill-formed bytes to the output as is, as every version of `to_bjdata` did before + this parameter was added; `strict` throws; `replace`/`ignore` sanitize it the same way [`dump`](dump.md) would. + If [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled, the default is `strict` instead. + ## Return value 1. BJData serialization as byte vector @@ -56,9 +65,9 @@ Strong guarantee: if an exception is thrown, there are no changes in the JSON va - Throws [`other_error.502`](../../home/exceptions.md#jsonexceptionother_error502) if `use_type` is true and `use_size` is false, and `j` contains a non-empty array, object, or binary value. -- Throws [type_error.316](../../home/exceptions.md#jsonexceptiontype_error316) if a string or object key in `j` is not - valid UTF-8 and [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled; otherwise, the bytes are - written unchanged +- Throws [type_error.316](../../home/exceptions.md#jsonexceptiontype_error316) if a string or object key in `j` is + not valid UTF-8 and `error_handler` is `strict` (the default only if + [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled) ## Complexity @@ -108,5 +117,6 @@ Linear in the size of the JSON value `j`. - Added in version 3.11.0. - BJData version parameter (for draft3 binary encoding) added in version 3.12.0. -- Throwing `type_error.316` for a string or object key that is not valid UTF-8 if - [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled added in version 3.13.0. \ No newline at end of file +- Added `error_handler` parameter in version 3.13.0. Its default, `keep`, writes the bytes of a string or object key + that is not valid UTF-8 unchanged, as before; `strict` (the default if + [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled) throws `type_error.316`. \ No newline at end of file diff --git a/docs/mkdocs/docs/api/basic_json/to_bson.md b/docs/mkdocs/docs/api/basic_json/to_bson.md index e3104b13c..ad974e048 100644 --- a/docs/mkdocs/docs/api/basic_json/to_bson.md +++ b/docs/mkdocs/docs/api/basic_json/to_bson.md @@ -2,11 +2,14 @@ ```cpp // (1) -static std::vector to_bson(const basic_json& j); +static std::vector to_bson(const basic_json& j, + const error_handler_t error_handler = error_handler_t::keep); // (2) -static void to_bson(const basic_json& j, detail::output_adapter o); -static void to_bson(const basic_json& j, detail::output_adapter o); +static void to_bson(const basic_json& j, detail::output_adapter o, + const error_handler_t error_handler = error_handler_t::keep); +static void to_bson(const basic_json& j, detail::output_adapter o, + const error_handler_t error_handler = error_handler_t::keep); ``` BSON (Binary JSON) is a binary format in which zero or more ordered key/value pairs are stored as a single entity (a @@ -25,6 +28,12 @@ The exact mapping and its limitations are described on a [dedicated page](../../ `o` (in) : output adapter to write serialization to +`error_handler` (in) +: how to treat a string or object key in `j` that is not valid UTF-8; see [`error_handler_t`](error_handler_t.md). + The default, `keep`, writes the ill-formed bytes to the output as is, as every version of `to_bson` did before + this parameter was added; `strict` throws; `replace`/`ignore` sanitize it the same way [`dump`](dump.md) would. + If [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled, the default is `strict` instead. + ## Return value 1. BSON serialization as a byte vector @@ -46,9 +55,9 @@ Strong guarantee: if an exception is thrown, there are no changes in the JSON va - Throws [`out_of_range.415`](../../home/exceptions.md#jsonexceptionout_of_range415) if the subtype of a binary value exceeds 255, the maximum of the BSON binary subtype; example: `"subtype 70000 is too large for the BSON binary subtype (max 255)"` -- Throws [type_error.316](../../home/exceptions.md#jsonexceptiontype_error316) if a string or object key is not valid - UTF-8 and [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled; otherwise, the bytes are - written unchanged +- Throws [type_error.316](../../home/exceptions.md#jsonexceptiontype_error316) if a string or object key is + not valid UTF-8 and `error_handler` is `strict` (the default only if + [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled) ## Complexity @@ -101,6 +110,7 @@ pass before anything is written. - Throws `out_of_range.412` and `out_of_range.415` since version 3.13.0. - Linear in the size of `j`, and no longer limited by the call stack for deeply nested values, since version 3.13.0. - `out_of_range.415` is now detected before anything is written, like the other exceptions above, since version 3.13.0. -- Throwing `type_error.316` for a string value or object key that is not valid UTF-8 if - [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled, detected before anything is written, - added in version 3.13.0. +- Added `error_handler` parameter in version 3.13.0. Its default, `keep`, writes the bytes of a string or object key + that is not valid UTF-8 unchanged, as before; `strict` (the default if + [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled) throws `type_error.316` before anything + is written. diff --git a/docs/mkdocs/docs/api/basic_json/to_cbor.md b/docs/mkdocs/docs/api/basic_json/to_cbor.md index cac8a0917..c72f310bb 100644 --- a/docs/mkdocs/docs/api/basic_json/to_cbor.md +++ b/docs/mkdocs/docs/api/basic_json/to_cbor.md @@ -2,11 +2,14 @@ ```cpp // (1) -static std::vector to_cbor(const basic_json& j); +static std::vector to_cbor(const basic_json& j, + const error_handler_t error_handler = error_handler_t::keep); // (2) -static void to_cbor(const basic_json& j, detail::output_adapter o); -static void to_cbor(const basic_json& j, detail::output_adapter o); +static void to_cbor(const basic_json& j, detail::output_adapter o, + const error_handler_t error_handler = error_handler_t::keep); +static void to_cbor(const basic_json& j, detail::output_adapter o, + const error_handler_t error_handler = error_handler_t::keep); ``` Serializes a given JSON value `j` to a byte vector using the CBOR (Concise Binary Object Representation) serialization @@ -26,6 +29,12 @@ The exact mapping and its limitations are described on a [dedicated page](../../ `o` (in) : output adapter to write serialization to +`error_handler` (in) +: how to treat a string or object key in `j` that is not valid UTF-8; see [`error_handler_t`](error_handler_t.md). + The default, `keep`, writes the ill-formed bytes to the output as is, as every version of `to_cbor` did before + this parameter was added; `strict` throws; `replace`/`ignore` sanitize it the same way [`dump`](dump.md) would. + If [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled, the default is `strict` instead. + ## Return value 1. CBOR serialization as a byte vector @@ -37,9 +46,9 @@ Strong guarantee: if an exception is thrown, there are no changes in the JSON va ## Exceptions -- Throws [type_error.316](../../home/exceptions.md#jsonexceptiontype_error316) if a string or object key in `j` is not - valid UTF-8 and [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled; otherwise, the bytes are - written unchanged +- Throws [type_error.316](../../home/exceptions.md#jsonexceptiontype_error316) if a string or object key in `j` is + not valid UTF-8 and `error_handler` is `strict` (the default only if + [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled) ## Complexity @@ -74,5 +83,6 @@ Linear in the size of the JSON value `j`. - Added in version 2.0.9. - Compact representation of floating-point numbers added in version 3.8.0. -- Throwing `type_error.316` for a string or object key that is not valid UTF-8 if - [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled added in version 3.13.0. +- Added `error_handler` parameter in version 3.13.0. Its default, `keep`, writes the bytes of a string or object key + that is not valid UTF-8 unchanged, as before; `strict` (the default if + [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled) throws `type_error.316`. diff --git a/docs/mkdocs/docs/api/basic_json/to_msgpack.md b/docs/mkdocs/docs/api/basic_json/to_msgpack.md index 707de9514..17b9fe29c 100644 --- a/docs/mkdocs/docs/api/basic_json/to_msgpack.md +++ b/docs/mkdocs/docs/api/basic_json/to_msgpack.md @@ -2,11 +2,14 @@ ```cpp // (1) -static std::vector to_msgpack(const basic_json& j); +static std::vector to_msgpack(const basic_json& j, + const error_handler_t error_handler = error_handler_t::keep); // (2) -static void to_msgpack(const basic_json& j, detail::output_adapter o); -static void to_msgpack(const basic_json& j, detail::output_adapter o); +static void to_msgpack(const basic_json& j, detail::output_adapter o, + const error_handler_t error_handler = error_handler_t::keep); +static void to_msgpack(const basic_json& j, detail::output_adapter o, + const error_handler_t error_handler = error_handler_t::keep); ``` Serializes a given JSON value `j` to a byte vector using the MessagePack serialization format. MessagePack is a binary @@ -25,6 +28,13 @@ The exact mapping and its limitations are described on a [dedicated page](../../ `o` (in) : output adapter to write serialization to +`error_handler` (in) +: how to treat a string or object key in `j` that is not valid UTF-8; see [`error_handler_t`](error_handler_t.md). + The default, `keep`, writes the ill-formed bytes to the output as is, as every version of `to_msgpack` did before + this parameter was added and as the MessagePack specification allows; `strict` throws; `replace`/`ignore` sanitize + it the same way [`dump`](dump.md) would. Unlike the other binary writers, the default stays `keep` even if + [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled. + ## Return value 1. MessagePack serialization as a byte vector @@ -42,6 +52,8 @@ Strong guarantee: if an exception is thrown, there are no changes in the JSON va - Throws [`out_of_range.415`](../../home/exceptions.md#jsonexceptionout_of_range415) if the subtype of a binary value exceeds 255, the maximum of the MessagePack ext type; example: `"subtype 70000 is too large for the MessagePack ext type (max 255)"` +- Throws [type_error.316](../../home/exceptions.md#jsonexceptiontype_error316) if a string or object key in `j` is + not valid UTF-8 and `error_handler` is `strict` ## Complexity @@ -91,6 +103,8 @@ Linear in the size of the JSON value `j`. - Added in version 2.0.9. - Throws `out_of_range.412` and `out_of_range.415` since version 3.13.0. +- Added `error_handler` parameter in version 3.13.0. Its default, `keep`, writes the bytes of a string or object key + that is not valid UTF-8 unchanged, as before. - Fixed in version 3.13.0 to serialize `number_integer_t`/`number_unsigned_t` pairs of different width correctly; before, integers could be serialized with the wrong value if `number_integer_t` was narrower than `number_unsigned_t`. diff --git a/docs/mkdocs/docs/api/basic_json/to_ubjson.md b/docs/mkdocs/docs/api/basic_json/to_ubjson.md index 8f2ea5eb9..6860c01c6 100644 --- a/docs/mkdocs/docs/api/basic_json/to_ubjson.md +++ b/docs/mkdocs/docs/api/basic_json/to_ubjson.md @@ -4,13 +4,16 @@ // (1) static std::vector to_ubjson(const basic_json& j, const bool use_size = false, - const bool use_type = false); + const bool use_type = false, + const error_handler_t error_handler = error_handler_t::keep); // (2) static void to_ubjson(const basic_json& j, detail::output_adapter o, - const bool use_size = false, const bool use_type = false); + const bool use_size = false, const bool use_type = false, + const error_handler_t error_handler = error_handler_t::keep); static void to_ubjson(const basic_json& j, detail::output_adapter o, - const bool use_size = false, const bool use_type = false); + const bool use_size = false, const bool use_type = false, + const error_handler_t error_handler = error_handler_t::keep); ``` Serializes a given JSON value `j` to a byte vector using the UBJSON (Universal Binary JSON) serialization format. UBJSON @@ -36,6 +39,12 @@ The exact mapping and its limitations are described on a [dedicated page](../../ : whether to add type annotations to container types (must be combined with `#!cpp use_size = true`); optional, `#!cpp false` by default. +`error_handler` (in) +: how to treat a string or object key in `j` that is not valid UTF-8; see [`error_handler_t`](error_handler_t.md). + The default, `keep`, writes the ill-formed bytes to the output as is, as every version of `to_ubjson` did before + this parameter was added; `strict` throws; `replace`/`ignore` sanitize it the same way [`dump`](dump.md) would. + If [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled, the default is `strict` instead. + ## Return value 1. UBJSON serialization as a byte vector @@ -49,9 +58,9 @@ Strong guarantee: if an exception is thrown, there are no changes in the JSON va - Throws [`other_error.502`](../../home/exceptions.md#jsonexceptionother_error502) if `use_type` is true and `use_size` is false, and `j` contains a non-empty array, object, or binary value. -- Throws [type_error.316](../../home/exceptions.md#jsonexceptiontype_error316) if a string or object key in `j` is not - valid UTF-8 and [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled; otherwise, the bytes are - written unchanged +- Throws [type_error.316](../../home/exceptions.md#jsonexceptiontype_error316) if a string or object key in `j` is + not valid UTF-8 and `error_handler` is `strict` (the default only if + [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled) ## Complexity @@ -100,5 +109,6 @@ Linear in the size of the JSON value `j`. ## Version history - Added in version 3.1.0. -- Throwing `type_error.316` for a string or object key that is not valid UTF-8 if - [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled added in version 3.13.0. +- Added `error_handler` parameter in version 3.13.0. Its default, `keep`, writes the bytes of a string or object key + that is not valid UTF-8 unchanged, as before; `strict` (the default if + [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled) throws `type_error.316`. diff --git a/docs/mkdocs/docs/api/macros/json_strict_binary_utf8.md b/docs/mkdocs/docs/api/macros/json_strict_binary_utf8.md index 7fe6d9a58..ed7bbe6b3 100644 --- a/docs/mkdocs/docs/api/macros/json_strict_binary_utf8.md +++ b/docs/mkdocs/docs/api/macros/json_strict_binary_utf8.md @@ -4,15 +4,18 @@ #define JSON_STRICT_BINARY_UTF8 /* value */ ``` -When defined to `1`, the binary writers [`to_cbor`](../basic_json/to_cbor.md), [`to_ubjson`](../basic_json/to_ubjson.md), -[`to_bjdata`](../basic_json/to_bjdata.md), and [`to_bson`](../basic_json/to_bson.md) check every string value and -object key for valid UTF-8 and throw [`type_error.316`](../../home/exceptions.md#jsonexceptiontype_error316) for -ill-formed UTF-8, like [`dump`](../basic_json/dump.md) does. Without it, they write the bytes unchanged. +When defined to `1`, the `error_handler` parameter of the binary writers [`to_cbor`](../basic_json/to_cbor.md), +[`to_ubjson`](../basic_json/to_ubjson.md), [`to_bjdata`](../basic_json/to_bjdata.md), and +[`to_bson`](../basic_json/to_bson.md) defaults to [`error_handler_t::strict`](../basic_json/error_handler_t.md) instead +of `error_handler_t::keep`. These writers then check every string value and object key for valid UTF-8 and throw +[`type_error.316`](../../home/exceptions.md#jsonexceptiontype_error316) for ill-formed UTF-8, like +[`dump`](../basic_json/dump.md) does. Without it, they write the bytes unchanged. An `error_handler` passed explicitly +always takes precedence. The macro does not affect: - [`to_msgpack`](../basic_json/to_msgpack.md): the MessagePack specification allows a `str` value to contain bytes that - are not valid UTF-8, so it always writes them unchanged. + are not valid UTF-8, so its `error_handler` always defaults to `keep`. - [`to_bon8`](../basic_json/to_bon8.md): BON8 always checks, because the UTF-8 lead bytes mark where a string ends. - The binary readers ([`from_cbor`](../basic_json/from_cbor.md), [`from_msgpack`](../basic_json/from_msgpack.md), [`from_ubjson`](../basic_json/from_ubjson.md), [`from_bjdata`](../basic_json/from_bjdata.md), @@ -33,8 +36,9 @@ The default value is `0` (disabled, the behavior of version 3.12.0 and earlier i CBOR, UBJSON, BJData, and BSON all require strings to be UTF-8. Up to version 3.12.0, the writers did not check this, so they could produce output that other decoders reject. Checking by default would break code that stores - other encodings (for instance ISO 8859-1) in a string and only ever writes it to a binary format, so this macro - offers the check as an opt-in ahead of version 4.0.0, where it is planned to become the default (see + other encodings (for instance ISO 8859-1) in a string and only ever writes it to a binary format. You can pass + `error_handler_t::strict` to each call, or use this macro to check by default ahead of version 4.0.0, where + `strict` is planned to become the default (see [#5529](https://github.com/nlohmann/json/issues/5529) and [#5651](https://github.com/nlohmann/json/issues/5651)). !!! warning "Opt-in only" diff --git a/docs/mkdocs/docs/examples/error_handler_t.cpp b/docs/mkdocs/docs/examples/error_handler_t.cpp index b4718d7e6..bf035cea3 100644 --- a/docs/mkdocs/docs/examples/error_handler_t.cpp +++ b/docs/mkdocs/docs/examples/error_handler_t.cpp @@ -20,5 +20,6 @@ int main() << j_invalid.dump(-1, ' ', false, json::error_handler_t::replace) << "\nstring with ignored invalid characters: " << j_invalid.dump(-1, ' ', false, json::error_handler_t::ignore) - << '\n'; + << "\nstring with the invalid byte kept as is (" << j_invalid.dump(-1, ' ', false, json::error_handler_t::keep).size() + << " bytes, not valid UTF-8 itself)\n"; } diff --git a/docs/mkdocs/docs/examples/error_handler_t.output b/docs/mkdocs/docs/examples/error_handler_t.output index 718d62bee..37cae62a2 100644 --- a/docs/mkdocs/docs/examples/error_handler_t.output +++ b/docs/mkdocs/docs/examples/error_handler_t.output @@ -1,3 +1,4 @@ [json.exception.type_error.316] invalid UTF-8 byte at index 2: 0xA9 string with replaced invalid characters: "ä�ü" string with ignored invalid characters: "äü" +string with the invalid byte kept as is (7 bytes, not valid UTF-8 itself) diff --git a/docs/mkdocs/docs/features/binary_formats/bjdata.md b/docs/mkdocs/docs/features/binary_formats/bjdata.md index 12d1af51b..3e3d8e885 100644 --- a/docs/mkdocs/docs/features/binary_formats/bjdata.md +++ b/docs/mkdocs/docs/features/binary_formats/bjdata.md @@ -65,10 +65,12 @@ The library uses the following mapping from JSON values types to BJData types ac !!! warning "UTF-8 validation of string values and object keys" - BJData strings must use UTF-8 encoding. By default, `to_bjdata()` writes the bytes of string values and object keys - unchanged, even if they are not valid UTF-8. If - [`JSON_STRICT_BINARY_UTF8`](../../api/macros/json_strict_binary_utf8.md) is enabled, it throws - [`type_error.316`](../../home/exceptions.md#jsonexceptiontype_error316) for ill-formed UTF-8 instead. + BJData strings must use UTF-8 encoding. By default (the [`error_handler`](../../api/basic_json/to_bjdata.md) + parameter left at `keep`), `to_bjdata()` writes the bytes of string values and object keys unchanged, even if they + are not valid UTF-8. With `error_handler_t::strict`, it throws + [`type_error.316`](../../home/exceptions.md#jsonexceptiontype_error316) for ill-formed UTF-8 instead; + `replace`/`ignore` sanitize the string. [`JSON_STRICT_BINARY_UTF8`](../../api/macros/json_strict_binary_utf8.md) + makes `strict` the default. !!! info "Unused BJData markers" @@ -217,12 +219,16 @@ The library maps BJData types to JSON value types as follows: !!! warning "Ill-formed UTF-8 in string values and object keys" - BJData strings must use UTF-8 encoding, but this is not enforced on read: `from_bjdata()` accepts a string - value or object key whose bytes are not valid UTF-8 and hands them back unchanged. However, + BJData strings must use UTF-8 encoding, but checking it on read is opt-in: with the + [`error_handler`](../../api/basic_json/from_bjdata.md) parameter left at `keep` (the default), `from_bjdata()` + accepts a string value or object key whose bytes are not valid UTF-8 and hands them back unchanged. Passing + `error_handler_t::strict` makes `from_bjdata()` check and throw + [`parse_error.113`](../../home/exceptions.md#jsonexceptionparse_error113) for ill-formed UTF-8, and + `replace`/`ignore` sanitize the string instead of keeping it. However, [`dump()`](../../api/basic_json/dump.md) still requires valid UTF-8 and throws - [`type_error.316`](../../home/exceptions.md#jsonexceptiontype_error316) for such a value, unless an error - handler is passed that replaces or ignores the ill-formed bytes. By default, `to_bjdata()` writes such a value - back unchanged (see above). + [`type_error.316`](../../home/exceptions.md#jsonexceptiontype_error316) for a value read with the default + `keep` handler, unless an error handler is passed that replaces or ignores the ill-formed bytes. `to_bjdata()`'s + own `error_handler` parameter defaults to `keep` (see above), so such a value is written back unchanged. !!! info "Round trips" diff --git a/docs/mkdocs/docs/features/binary_formats/bson.md b/docs/mkdocs/docs/features/binary_formats/bson.md index 245ed3ebf..f205cba17 100644 --- a/docs/mkdocs/docs/features/binary_formats/bson.md +++ b/docs/mkdocs/docs/features/binary_formats/bson.md @@ -112,13 +112,18 @@ The library maps BSON record types to JSON value types as follows: !!! warning "Ill-formed UTF-8 in string values" The BSON specification requires `string` values (type `0x02`) to be valid UTF-8, but this is not required of a - decoder. `from_bson()` accepts a `string` value whose bytes are not valid UTF-8 and hands them back unchanged. - However, [`dump()`](../../api/basic_json/dump.md) still requires valid UTF-8 and throws - [`type_error.316`](../../home/exceptions.md#jsonexceptiontype_error316) for such a value, unless an error handler is - passed that replaces or ignores the ill-formed bytes. By default, `to_bson()` writes such a string value or element - (key) name unchanged; if [`JSON_STRICT_BINARY_UTF8`](../../api/macros/json_strict_binary_utf8.md) is enabled, it - throws the same exception instead. Element (key) names are never validated on read, since they are read byte-by-byte - as a C string. `binary` values (type `0x05`) are unaffected, since they are not required to hold text. + decoder, so checking is opt-in: with the [`error_handler`](../../api/basic_json/from_bson.md) parameter left at + `keep` (the default), `from_bson()` accepts a `string` value whose bytes are not valid UTF-8 and hands them back + unchanged. Passing `error_handler_t::strict` makes `from_bson()` check and throw + [`parse_error.113`](../../home/exceptions.md#jsonexceptionparse_error113) for ill-formed UTF-8, and + `replace`/`ignore` sanitize the string instead of keeping it. However, [`dump()`](../../api/basic_json/dump.md) + still requires valid UTF-8 and throws [`type_error.316`](../../home/exceptions.md#jsonexceptiontype_error316) for a + value read with the default `keep` handler, unless an error handler is passed that replaces or ignores the + ill-formed bytes. `to_bson()`'s own `error_handler` parameter defaults to `keep`, so such a string value or element + (key) name is written unchanged; with `strict` (the default if + [`JSON_STRICT_BINARY_UTF8`](../../api/macros/json_strict_binary_utf8.md) is enabled), it throws the same exception + instead. Element (key) names are never validated on read, since they are read byte-by-byte as a C string. `binary` + values (type `0x05`) are unaffected, since they are not required to hold text. ??? example "Example: deserialize a JSON value from BSON" diff --git a/docs/mkdocs/docs/features/binary_formats/cbor.md b/docs/mkdocs/docs/features/binary_formats/cbor.md index 66300c5fa..7b5be4631 100644 --- a/docs/mkdocs/docs/features/binary_formats/cbor.md +++ b/docs/mkdocs/docs/features/binary_formats/cbor.md @@ -192,12 +192,17 @@ The library maps CBOR types to JSON value types as follows: !!! warning "Ill-formed UTF-8 in text strings" [RFC 8949, Section 3.1](https://www.rfc-editor.org/rfc/rfc8949.html#section-3.1) requires CBOR text strings (major - type 3) to be valid UTF-8, but leaves it up to the decoder whether to enforce this. This library does not: - `from_cbor()` accepts a text string (object keys included) whose bytes are not valid UTF-8 and hands them back - unchanged. However, [`dump()`](../../api/basic_json/dump.md) still requires valid UTF-8 and throws - [`type_error.316`](../../home/exceptions.md#jsonexceptiontype_error316) for such a value, unless an error handler is - passed that replaces or ignores the ill-formed bytes. By default, `to_cbor()` writes such a value back unchanged; if - [`JSON_STRICT_BINARY_UTF8`](../../api/macros/json_strict_binary_utf8.md) is enabled, it throws the same exception + type 3) to be valid UTF-8, but leaves it up to the decoder whether to enforce this, so checking is opt-in: with the + [`error_handler`](../../api/basic_json/from_cbor.md) parameter left at `keep` (the default), `from_cbor()` accepts a + text string (object keys included) whose bytes are not valid UTF-8 and hands them back unchanged. Passing + `error_handler_t::strict` makes `from_cbor()` check and throw + [`parse_error.113`](../../home/exceptions.md#jsonexceptionparse_error113) for ill-formed UTF-8, and + `replace`/`ignore` sanitize the string instead of keeping it. However, [`dump()`](../../api/basic_json/dump.md) + still requires valid UTF-8 and throws [`type_error.316`](../../home/exceptions.md#jsonexceptiontype_error316) for a + value read with the default `keep` handler, unless an error handler is passed that replaces or ignores the + ill-formed bytes. `to_cbor()`'s own [`error_handler`](../../api/basic_json/to_cbor.md) parameter defaults to `keep`, + so such a value is written back unchanged; with `strict` (the default if + [`JSON_STRICT_BINARY_UTF8`](../../api/macros/json_strict_binary_utf8.md) is enabled), it throws the same exception instead. Byte strings (major type 2) are unaffected, since they are not required to hold text. !!! warning "Tagged items" diff --git a/docs/mkdocs/docs/features/binary_formats/messagepack.md b/docs/mkdocs/docs/features/binary_formats/messagepack.md index 047944852..24eeab139 100644 --- a/docs/mkdocs/docs/features/binary_formats/messagepack.md +++ b/docs/mkdocs/docs/features/binary_formats/messagepack.md @@ -157,11 +157,19 @@ The library maps MessagePack types to JSON value types as follows: The MessagePack specification explicitly allows a `str` value (`fixstr`, `str 8`, `str 16`, `str 32`) to contain a byte sequence that is not valid UTF-8, and expects a deserializer to hand the original bytes back unchanged. - This library follows that: `from_msgpack()` reads `str` bytes (object keys included) as-is, without validating - them, and `to_msgpack()` writes them back as-is, so such a value round-trips through `from_msgpack(to_msgpack(j))` - byte for byte. However, [`dump()`](../../api/basic_json/dump.md) still requires valid UTF-8 and throws - [`type_error.316`](../../home/exceptions.md#jsonexceptiontype_error316) for a value read this way, unless an - error handler is passed that replaces or ignores the ill-formed bytes. + This library follows that by default: with its + [`error_handler`](../../api/basic_json/from_msgpack.md) parameter left at `keep` (the default), + `from_msgpack()` reads `str` bytes (object keys included) as-is, without validating them, so such a value + round-trips through `from_msgpack(to_msgpack(j))` byte for byte. Passing `error_handler_t::strict` makes + `from_msgpack()` check anyway and throw + [`parse_error.113`](../../home/exceptions.md#jsonexceptionparse_error113) for ill-formed UTF-8, and + `replace`/`ignore` sanitize the string instead of keeping it. `to_msgpack()` also writes `str` bytes as-is by + default, since the specification permits it; its [`error_handler`](../../api/basic_json/to_msgpack.md) parameter + can be set to `strict` to throw [`type_error.316`](../../home/exceptions.md#jsonexceptiontype_error316) instead, or + to `replace`/`ignore` to sanitize the string, for instance for a decoder that rejects ill-formed UTF-8. However, + [`dump()`](../../api/basic_json/dump.md) still requires valid UTF-8 and throws + [`type_error.316`](../../home/exceptions.md#jsonexceptiontype_error316) for a value read this way with the + default `keep` handler, unless an error handler is passed that replaces or ignores the ill-formed bytes. ??? example "Example: deserialize a JSON value from MessagePack" diff --git a/docs/mkdocs/docs/features/binary_formats/ubjson.md b/docs/mkdocs/docs/features/binary_formats/ubjson.md index dbdff6e6c..f7a14d855 100644 --- a/docs/mkdocs/docs/features/binary_formats/ubjson.md +++ b/docs/mkdocs/docs/features/binary_formats/ubjson.md @@ -49,10 +49,12 @@ The library uses the following mapping from JSON values types to UBJSON types ac !!! warning "UTF-8 validation of string values and object keys" - UBJSON's required string encoding is UTF-8. By default, `to_ubjson()` writes the bytes of string values and object - keys unchanged, even if they are not valid UTF-8. If - [`JSON_STRICT_BINARY_UTF8`](../../api/macros/json_strict_binary_utf8.md) is enabled, it throws - [`type_error.316`](../../home/exceptions.md#jsonexceptiontype_error316) for ill-formed UTF-8 instead. + UBJSON's required string encoding is UTF-8. By default (the [`error_handler`](../../api/basic_json/to_ubjson.md) + parameter left at `keep`), `to_ubjson()` writes the bytes of string values and object keys unchanged, even if they + are not valid UTF-8. With `error_handler_t::strict`, it throws + [`type_error.316`](../../home/exceptions.md#jsonexceptiontype_error316) for ill-formed UTF-8 instead; + `replace`/`ignore` sanitize the string. [`JSON_STRICT_BINARY_UTF8`](../../api/macros/json_strict_binary_utf8.md) + makes `strict` the default. !!! info "Unused UBJSON markers" @@ -129,12 +131,16 @@ The library maps UBJSON types to JSON value types as follows: !!! warning "Ill-formed UTF-8 in string values and object keys" - UBJSON's required string encoding is UTF-8, but this is not enforced on read: `from_ubjson()` accepts a string - value or object key whose bytes are not valid UTF-8 and hands them back unchanged. However, + UBJSON's required string encoding is UTF-8, but checking it on read is opt-in: with the + [`error_handler`](../../api/basic_json/from_ubjson.md) parameter left at `keep` (the default), `from_ubjson()` + accepts a string value or object key whose bytes are not valid UTF-8 and hands them back unchanged. Passing + `error_handler_t::strict` makes `from_ubjson()` check and throw + [`parse_error.113`](../../home/exceptions.md#jsonexceptionparse_error113) for ill-formed UTF-8, and + `replace`/`ignore` sanitize the string instead of keeping it. However, [`dump()`](../../api/basic_json/dump.md) still requires valid UTF-8 and throws - [`type_error.316`](../../home/exceptions.md#jsonexceptiontype_error316) for such a value, unless an error - handler is passed that replaces or ignores the ill-formed bytes. By default, `to_ubjson()` writes such a value - back unchanged (see above). + [`type_error.316`](../../home/exceptions.md#jsonexceptiontype_error316) for a value read with the default + `keep` handler, unless an error handler is passed that replaces or ignores the ill-formed bytes. `to_ubjson()`'s + own `error_handler` parameter defaults to `keep` (see above), so such a value is written back unchanged. ??? example "Example: deserialize a JSON value from UBJSON" diff --git a/docs/mkdocs/docs/home/exceptions.md b/docs/mkdocs/docs/home/exceptions.md index a3797cf04..e7eb8fd03 100644 --- a/docs/mkdocs/docs/home/exceptions.md +++ b/docs/mkdocs/docs/home/exceptions.md @@ -340,9 +340,11 @@ An unexpected byte was read in a [binary format](../features/binary_formats/inde ### json.exception.parse_error.113 A string could not be read from a [binary format](../features/binary_formats/index.md): either a value that is not a -string was read where one was required (for instance as a map key), or the string's length specification is invalid. -The bytes of a string itself are not checked for valid UTF-8 on read; see the ill-formed UTF-8 notes on the -individual [binary format](../features/binary_formats/index.md) pages for how such a string is handled afterward. +string was read where one was required (for instance as a map key), the string's length specification is invalid, or +the string's bytes are not valid UTF-8 and the `error_handler` parameter of the corresponding `from_*` function is +set to `strict`. By default (`error_handler_t::keep`), the bytes of a string are not checked for valid UTF-8 on read; +see the ill-formed UTF-8 notes on the individual [binary format](../features/binary_formats/index.md) pages for how +such a string is handled depending on `error_handler`. CBOR and MessagePack allow map keys of any type, but JSON object keys are always strings. Maps with keys of any other type (for instance integers or `null`) are therefore not supported; see the notes on @@ -365,6 +367,9 @@ type (for instance integers or `null`) are therefore not supported; see the note ``` [json.exception.parse_error.113] parse error at byte 3: syntax error while parsing BJData string: string length must not be negative ``` + ``` + [json.exception.parse_error.113] parse error at byte 3: syntax error while parsing CBOR string: invalid string: ill-formed UTF-8 byte + ``` ### json.exception.parse_error.114 @@ -747,10 +752,11 @@ The [`unflatten()`](../api/basic_json/unflatten.md) function only works for an o The [`dump()`](../api/basic_json/dump.md) function only works with UTF-8 encoded strings; that is, if you assign a `std::string` to a JSON value, make sure it is UTF-8 encoded. See the FAQ entry on [serializing untrusted or invalid UTF-8](faq.md#serializing-untrusted-or-invalid-utf-8) for background and the recommended fix. -If [`JSON_STRICT_BINARY_UTF8`](../api/macros/json_strict_binary_utf8.md) is enabled, the binary writers -[`to_cbor()`](../api/basic_json/to_cbor.md), [`to_ubjson()`](../api/basic_json/to_ubjson.md), +The binary writers [`to_cbor()`](../api/basic_json/to_cbor.md), [`to_ubjson()`](../api/basic_json/to_ubjson.md), [`to_bjdata()`](../api/basic_json/to_bjdata.md), and [`to_bson()`](../api/basic_json/to_bson.md) throw this exception -for a string value or object key that is not valid UTF-8 as well. +as well for a string value or object key that is not valid UTF-8 if their `error_handler` is `strict` (the default if +[`JSON_STRICT_BINARY_UTF8`](../api/macros/json_strict_binary_utf8.md) is enabled). So does +[`to_msgpack()`](../api/basic_json/to_msgpack.md) if `error_handler_t::strict` is passed. !!! failure "Example message" diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index 3d79ad41b..1a78a6502 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -108,8 +109,16 @@ class binary_reader @brief create a binary reader @param[in] adapter input adapter to read from + @param[in] format the binary format to parse + @param[in] error_handler how to treat text strings and object keys that + are not well-formed UTF-8; none of the supported formats + requires a decoder to reject those, so the default is to + @ref error_handler_t::keep them unchanged, as every binary + reader did before this parameter existed */ - explicit binary_reader(InputAdapterType&& adapter, const input_format_t format = input_format_t::json) noexcept : ia(std::move(adapter)), input_format(format) + explicit binary_reader(InputAdapterType&& adapter, const input_format_t format = input_format_t::json, + const error_handler_t error_handler = error_handler_t::keep) noexcept + : ia(std::move(adapter)), input_format(format), error_handler(error_handler) { (void)detail::is_sax_static_asserts {}; } @@ -428,7 +437,7 @@ class binary_reader { if (get_bson_cstr_bulk(result, std::integral_constant {})) { - return true; + return check_string_utf8(result, "key"); } auto out = std::back_inserter(result); @@ -441,7 +450,7 @@ class binary_reader } if (current == 0x00) { - return true; + return check_string_utf8(result, "key"); } *out++ = static_cast(current); } @@ -522,7 +531,7 @@ class binary_reader "string"), nullptr)); } - return true; + return check_string_utf8(result, "string"); } /*! @@ -1149,7 +1158,7 @@ class binary_reader @return whether string creation completed */ - bool get_cbor_string(string_t& result) + bool get_cbor_string(string_t& result, const char* context = "string") { // number of indefinite-length strings that have been opened and not // closed yet. RFC 8949, Section 3.2.3 does not permit nesting them, @@ -1179,7 +1188,7 @@ class binary_reader { if (--open == 0) { - return true; + return check_string_utf8(result, context); } get(); continue; @@ -1192,7 +1201,7 @@ class binary_reader if (open == 0) { - return true; + return check_string_utf8(result, context); } get(); @@ -1216,7 +1225,7 @@ class binary_reader // EOF and major type 3 (text string) are left to get_cbor_string if (current == char_traits::eof() || (static_cast(current) & 0xE0u) == 0x60u) { - return get_cbor_string(result); + return get_cbor_string(result, "key"); } const char* found = nullptr; @@ -2004,7 +2013,7 @@ class binary_reader @return whether string creation completed */ - bool get_msgpack_string(string_t& result) + bool get_msgpack_string(string_t& result, const char* context = "string") { if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::msgpack, "string"))) { @@ -2047,25 +2056,25 @@ class binary_reader case 0xBE: case 0xBF: { - return get_string(input_format_t::msgpack, static_cast(current) & 0x1Fu, result); + return get_string(input_format_t::msgpack, static_cast(current) & 0x1Fu, result) && check_string_utf8(result, context); } case 0xD9: // str 8 { std::uint8_t len{}; - return get_number(input_format_t::msgpack, len) && get_string(input_format_t::msgpack, len, result); + return get_number(input_format_t::msgpack, len) && get_string(input_format_t::msgpack, len, result) && check_string_utf8(result, context); } case 0xDA: // str 16 { std::uint16_t len{}; - return get_number(input_format_t::msgpack, len) && get_string(input_format_t::msgpack, len, result); + return get_number(input_format_t::msgpack, len) && get_string(input_format_t::msgpack, len, result) && check_string_utf8(result, context); } case 0xDB: // str 32 { std::uint32_t len{}; - return get_number(input_format_t::msgpack, len) && get_string(input_format_t::msgpack, len, result); + return get_number(input_format_t::msgpack, len) && get_string(input_format_t::msgpack, len, result) && check_string_utf8(result, context); } default: @@ -2143,7 +2152,7 @@ class binary_reader // byte 0xC1 are left to get_msgpack_string if (current == char_traits::eof()) { - return get_msgpack_string(result); + return get_msgpack_string(result, "key"); } if (current <= 0x7F || current >= 0xE0) { @@ -2159,7 +2168,7 @@ class binary_reader } else { - return get_msgpack_string(result); + return get_msgpack_string(result, "key"); } break; } @@ -2405,7 +2414,7 @@ class binary_reader if (top.is_object) { key.clear(); - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key))) + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key, true, "key") || !sax->key(key))) { return false; } @@ -2427,7 +2436,7 @@ class binary_reader if (top.is_object) { key.clear(); - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key, false) || !sax->key(key))) + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key, false, "key") || !sax->key(key))) { return false; } @@ -2495,7 +2504,7 @@ class binary_reader @return whether string creation completed */ - bool get_ubjson_string(string_t& result, const bool get_char = true) + bool get_ubjson_string(string_t& result, const bool get_char = true, const char* context = "string") { if (get_char) { @@ -2516,31 +2525,31 @@ class binary_reader case 'U': { std::uint8_t len{}; - return get_number(input_format, len) && get_string(input_format, len, result); + return get_number(input_format, len) && get_string(input_format, len, result) && check_string_utf8(result, context); } case 'i': { std::int8_t len{}; - return get_number(input_format, len) && check_ubjson_string_length(len) && get_string(input_format, len, result); + return get_number(input_format, len) && check_ubjson_string_length(len) && get_string(input_format, len, result) && check_string_utf8(result, context); } case 'I': { std::int16_t len{}; - return get_number(input_format, len) && check_ubjson_string_length(len) && get_string(input_format, len, result); + return get_number(input_format, len) && check_ubjson_string_length(len) && get_string(input_format, len, result) && check_string_utf8(result, context); } case 'l': { std::int32_t len{}; - return get_number(input_format, len) && check_ubjson_string_length(len) && get_string(input_format, len, result); + return get_number(input_format, len) && check_ubjson_string_length(len) && get_string(input_format, len, result) && check_string_utf8(result, context); } case 'L': { std::int64_t len{}; - return get_number(input_format, len) && check_ubjson_string_length(len) && get_string(input_format, len, result); + return get_number(input_format, len) && check_ubjson_string_length(len) && get_string(input_format, len, result) && check_string_utf8(result, context); } case 'u': @@ -2550,7 +2559,7 @@ class binary_reader break; } std::uint16_t len{}; - return get_number(input_format, len) && get_string(input_format, len, result); + return get_number(input_format, len) && get_string(input_format, len, result) && check_string_utf8(result, context); } case 'm': @@ -2560,7 +2569,7 @@ class binary_reader break; } std::uint32_t len{}; - return get_number(input_format, len) && get_string(input_format, len, result); + return get_number(input_format, len) && get_string(input_format, len, result) && check_string_utf8(result, context); } case 'M': @@ -2570,7 +2579,7 @@ class binary_reader break; } std::uint64_t len{}; - return get_number(input_format, len) && get_string(input_format, len, result); + return get_number(input_format, len) && get_string(input_format, len, result) && check_string_utf8(result, context); } default: @@ -4044,15 +4053,53 @@ class binary_reader const NumberType len, string_t& result) { - // Strings are taken as is: none of CBOR (RFC 8949 §3.1 leaves the - // choice to the decoder), MessagePack (whose spec explicitly allows - // a str object to contain an invalid byte sequence), UBJSON, BJData, - // or BSON requires a decoder to reject ill-formed UTF-8. The bytes - // are kept unchanged; dump() and the binary writers are the ones - // that check them and report type_error.316 if they are not valid. + // Strings are taken as is by default: none of CBOR (RFC 8949 §3.1 + // leaves the choice to the decoder), MessagePack (whose spec + // explicitly allows a str object to contain an invalid byte + // sequence), UBJSON, BJData, or BSON requires a decoder to reject + // ill-formed UTF-8. Checking (and, with @ref error_handler_t::strict, + // rejecting, or with `replace`/`ignore`, sanitizing) is opt-in via + // @ref error_handler, applied once the whole string (all chunks of + // an indefinite-length CBOR string included) has been assembled, by + // @ref check_string_utf8 at the call site. return get_bytes(format, len, "string", result); } + /*! + @brief validate a decoded text string (value or object key) against @ref error_handler + + None of the binary formats requires a decoder to reject ill-formed UTF-8 + in a text string (see @ref get_string), so by default + (@ref error_handler_t::keep) this does nothing. A stricter + @ref error_handler opts into the same well-formedness check @ref + serializer::dump_escaped_impl applies when dumping a string: + @ref error_handler_t::strict rejects ill-formed input with + parse_error.113 (honoring `allow_exceptions` via @a sax), while + @ref error_handler_t::replace / @ref error_handler_t::ignore sanitize + @a result in place, using the exact same rules. + + @param[in,out] result the already assembled string to check + @param[in] context further context information (for diagnostics) + @return whether @a result is acceptable (always true for `keep`) + */ + bool check_string_utf8(string_t& result, const char* context) + { + if (error_handler == error_handler_t::keep || is_valid_utf8(result)) + { + return true; + } + + if (error_handler == error_handler_t::strict) + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, + exception_message(input_format, "invalid string: ill-formed UTF-8 byte", context), nullptr)); + } + + result = sanitize_utf8(result, error_handler); + return true; + } + /*! @brief create a byte array by reading bytes from the input @@ -4226,6 +4273,9 @@ class binary_reader /// input format const input_format_t input_format = input_format_t::json; + /// how to treat text strings/object keys that are not well-formed UTF-8 + const error_handler_t error_handler = error_handler_t::keep; + /// the SAX parser json_sax_t* sax = nullptr; diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index da5aa0cf4..e74671db2 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -93,8 +94,12 @@ class binary_writer @param[in] sink output sink to write to (a value-type sink such as output_vector_sink, or output_adapter_sink wrapping a type-erased output adapter) + @param[in] error_handler_ how to treat a string value or object key that + is not valid UTF-8 (CBOR, MessagePack, UBJSON, BJData, and BSON; + never consulted by @ref write_bon8) */ - explicit binary_writer(OutputSinkType sink) : oa(std::move(sink)) + explicit binary_writer(OutputSinkType sink, const error_handler_t error_handler_ = binary_writer_default_error_handler()) + : oa(std::move(sink)), error_handler(error_handler_) {} /*! @@ -107,16 +112,20 @@ class binary_writer from one. @param[in] adapter output adapter to write to + @param[in] error_handler_ how to treat a string value or object key that + is not valid UTF-8 (CBOR, MessagePack, UBJSON, BJData, and BSON; + never consulted by @ref write_bon8) */ template < typename SinkType = OutputSinkType, typename std::enable_if < std::is_constructible>::value, int >::type = 0 > - explicit binary_writer(output_adapter_t adapter) : oa(SinkType(std::move(adapter))) + explicit binary_writer(output_adapter_t adapter, const error_handler_t error_handler_ = binary_writer_default_error_handler()) + : oa(SinkType(std::move(adapter))), error_handler(error_handler_) {} /*! @param[in] j JSON value to serialize - @throw type_error.316 if JSON_STRICT_BINARY_UTF8 is enabled and a string - value or an object key is not valid UTF-8 + @throw type_error.316 if a string value or an object key is not valid + UTF-8 @throw type_error.317 if @a j is not an object */ void write_bson(const BasicJsonType& j) @@ -147,8 +156,8 @@ class binary_writer /*! @param[in] j JSON value to serialize - @throw type_error.316 if JSON_STRICT_BINARY_UTF8 is enabled and a string - value or an object key is not valid UTF-8 + @throw type_error.316 if a string value or an object key is not valid + UTF-8 */ void write_cbor(const BasicJsonType& j) { @@ -215,15 +224,16 @@ class binary_writer case value_t::string: { - check_text_utf8(*j.m_data.m_value.string, j); + string_t storage; + const string_t& value = sanitize_utf8_for_write(*j.m_data.m_value.string, j, storage); // step 1: write control byte and the string length - write_cbor_head(0x60, j.m_data.m_value.string->size()); + write_cbor_head(0x60, value.size()); // step 2: write the string oa.write_characters( - reinterpret_cast(j.m_data.m_value.string->data()), - j.m_data.m_value.string->size()); + reinterpret_cast(value.data()), + value.size()); break; } @@ -296,8 +306,14 @@ class binary_writer // el.first is checked here, against the object as // diagnostics context, because write_cbor(el.first) // converts it to a temporary basic_json that would be - // used as the context instead - check_text_utf8(el.first, j); + // used as the context instead; for error_handler_t::keep + // and ::replace/::ignore the recursive write_cbor(el.first) + // call below handles the key like any other string, so no + // separate check is needed here for those + if (error_handler == error_handler_t::strict) + { + check_utf8(el.first, j); + } write_cbor(el.first); write_cbor(el.second); } @@ -445,8 +461,11 @@ class binary_writer case value_t::string: { + string_t storage; + const string_t& value = sanitize_utf8_for_write(*j.m_data.m_value.string, j, storage); + // step 1: write control byte and the string length - const auto N = to_msgpack_length(j.m_data.m_value.string->size(), j); + const auto N = to_msgpack_length(value.size(), j); if (N <= 31) { // fixstr @@ -473,8 +492,8 @@ class binary_writer // step 2: write the string oa.write_characters( - reinterpret_cast(j.m_data.m_value.string->data()), - j.m_data.m_value.string->size()); + reinterpret_cast(value.data()), + value.size()); break; } @@ -621,6 +640,13 @@ class binary_writer // step 2: write each element for (const auto& el : *j.m_data.m_value.object) { + // as in write_cbor, el.first is checked here against the + // object as diagnostics context; the recursive call below + // handles keep/replace/ignore like any other string + if (error_handler == error_handler_t::strict) + { + check_utf8(el.first, j); + } write_msgpack(el.first); write_msgpack(el.second); } @@ -640,8 +666,8 @@ class binary_writer @param[in] add_prefix whether prefixes need to be used for this value @param[in] use_bjdata whether write in BJData format, default is false @param[in] bjdata_version which BJData version to use, default is draft2 - @throw type_error.316 if JSON_STRICT_BINARY_UTF8 is enabled and a string - value or an object key is not valid UTF-8 + @throw type_error.316 if a string value or an object key is not valid + UTF-8 */ void write_ubjson(const BasicJsonType& j, const bool use_count, const bool use_type, const bool add_prefix = true, @@ -691,16 +717,17 @@ class binary_writer case value_t::string: { - check_text_utf8(*j.m_data.m_value.string, j); + string_t storage; + const string_t& value = sanitize_utf8_for_write(*j.m_data.m_value.string, j, storage); if (add_prefix) { oa.write_character(to_char_type('S')); } - write_number_with_ubjson_prefix(j.m_data.m_value.string->size(), true, use_bjdata); + write_number_with_ubjson_prefix(value.size(), true, use_bjdata); oa.write_characters( - reinterpret_cast(j.m_data.m_value.string->data()), - j.m_data.m_value.string->size()); + reinterpret_cast(value.data()), + value.size()); break; } @@ -855,11 +882,12 @@ class binary_writer for (const auto& el : *j.m_data.m_value.object) { - check_text_utf8(el.first, j); - write_number_with_ubjson_prefix(el.first.size(), true, use_bjdata); + string_t storage; + const string_t& key = sanitize_utf8_for_write(el.first, j, storage); + write_number_with_ubjson_prefix(key.size(), true, use_bjdata); oa.write_characters( - reinterpret_cast(el.first.data()), - el.first.size()); + reinterpret_cast(key.data()), + key.size()); write_ubjson(el.second, use_count, use_type, prefix_required, use_bjdata, bjdata_version); } @@ -902,10 +930,10 @@ class binary_writer and the entry name size (and its null-terminator). @throw out_of_range.409 if @a name contains U+0000, before anything is written - @throw type_error.316 if JSON_STRICT_BINARY_UTF8 is enabled and @a name is - not valid UTF-8, before anything is written + @throw type_error.316 if @a name is not valid UTF-8, before anything is + written */ - static std::size_t calc_bson_entry_header_size(const string_t& name, const BasicJsonType& j) + std::size_t calc_bson_entry_header_size(const string_t& name, const BasicJsonType& j) { const auto it = name.find(static_cast(0)); if (JSON_HEDLEY_UNLIKELY(it != BasicJsonType::string_t::npos)) @@ -913,9 +941,10 @@ class binary_writer JSON_THROW(out_of_range::create(409, concat("BSON key cannot contain code point U+0000 (at byte ", std::to_string(it), ")"), &j)); } - check_text_utf8(name, j); + string_t storage; + const string_t& sanitized = sanitize_utf8_for_write(name, j, storage); - return /*id*/ 1ul + name.size() + /*zero-terminator*/1u; + return /*id*/ 1ul + sanitized.size() + /*zero-terminator*/1u; } /*! @@ -935,14 +964,28 @@ class binary_writer /*! @brief Writes the given @a element_type and @a name to the output adapter + + @a name has already been validated (and, for @ref error_handler_t::strict, + found well-formed) by @ref calc_bson_entry_header_size during the earlier + size pass, so only @ref error_handler_t::replace / @ref + error_handler_t::ignore need to sanitize it again here, to actually write + the bytes that size was computed from. */ void write_bson_entry_header(const string_t& name, const std::uint8_t element_type) { oa.write_character(to_char_type(element_type)); - oa.write_characters( - reinterpret_cast(name.data()), - name.size()); + + if (error_handler == error_handler_t::keep || error_handler == error_handler_t::strict || is_valid_utf8(name)) + { + oa.write_characters(reinterpret_cast(name.data()), name.size()); + } + else + { + const string_t sanitized = sanitize_utf8(name, error_handler); + oa.write_characters(reinterpret_cast(sanitized.data()), sanitized.size()); + } + // the terminating null byte is written explicitly rather than taken // from the buffer, so that string_t::data() need not be null-terminated oa.write_character(to_char_type(0x00)); @@ -970,8 +1013,8 @@ class binary_writer /*! @return The size of the BSON-encoded string in @a value - @throw type_error.316 if JSON_STRICT_BINARY_UTF8 is enabled and @a value - is not valid UTF-8, before anything is written + @throw type_error.316 if @a value is not valid UTF-8, before anything is + written @note The UTF-8 check is skipped if @a value is already too long for the 32-bit BSON length field (@ref to_bson_length rejects it later, once @@ -979,27 +1022,41 @@ class binary_writer from reading past a StringType that reports a size larger than what it actually holds. */ - static std::size_t calc_bson_string_size(const string_t& value, const BasicJsonType& j) + std::size_t calc_bson_string_size(const string_t& value, const BasicJsonType& j) { if (JSON_HEDLEY_LIKELY(value_in_range_of(value.size()))) { - check_text_utf8(value, j); + string_t storage; + const string_t& sanitized = sanitize_utf8_for_write(value, j, storage); + return sizeof(std::int32_t) + sanitized.size() + 1ul; } return sizeof(std::int32_t) + value.size() + 1ul; } /*! @brief Writes a BSON element with key @a name and string value @a value + + @a value has already been validated (and, for @ref error_handler_t::strict, + found well-formed) by @ref calc_bson_string_size during the earlier size + pass, so only @ref error_handler_t::replace / @ref error_handler_t::ignore + need to sanitize it again here, to actually write the bytes that size was + computed from. */ void write_bson_string(const string_t& name, const string_t& value) { write_bson_entry_header(name, 0x02); - write_number(to_bson_length(value.size() + 1ul), true); + const bool sanitize = error_handler != error_handler_t::keep + && error_handler != error_handler_t::strict + && !is_valid_utf8(value); + const string_t sanitized = sanitize ? sanitize_utf8(value, error_handler) : string_t{}; + const string_t& written = sanitize ? sanitized : value; + + write_number(to_bson_length(written.size() + 1ul), true); oa.write_characters( - reinterpret_cast(value.data()), - value.size()); + reinterpret_cast(written.data()), + written.size()); // the terminating null byte is written explicitly rather than taken // from the buffer, so that string_t::data() need not be null-terminated oa.write_character(to_char_type(0x00)); @@ -1113,10 +1170,10 @@ class binary_writer is neither an object nor an array @throw out_of_range.415 if @a j is binary with a subtype that does not fit into a byte, before anything is written - @throw type_error.316 if JSON_STRICT_BINARY_UTF8 is enabled and @a j is a - string that is not valid UTF-8, before anything is written + @throw type_error.316 if @a j is a string that is not valid UTF-8, before + anything is written */ - static std::size_t calc_bson_value_size(const BasicJsonType& j) + std::size_t calc_bson_value_size(const BasicJsonType& j) { switch (j.type()) { @@ -1249,10 +1306,10 @@ class binary_writer written @throw out_of_range.415 if a binary value's subtype does not fit into a byte, before anything is written - @throw type_error.316 if JSON_STRICT_BINARY_UTF8 is enabled and a string - value or a key is not valid UTF-8, before anything is written + @throw type_error.316 if a string value or a key is not valid UTF-8, + before anything is written */ - static std::size_t calc_bson_sizes(const BasicJsonType& document, std::vector& nested_sizes) + std::size_t calc_bson_sizes(const BasicJsonType& document, std::vector& nested_sizes) { // the object or array whose entries are being sized, and the ones it // is in; nothing is allocated unless the document nests @@ -2171,26 +2228,54 @@ class binary_writer } /*! - @brief check a CBOR, UBJSON, BJData, or BSON text string for valid UTF-8 + @brief return @a s as it should be written, honoring @ref error_handler - The check only happens if JSON_STRICT_BINARY_UTF8 is enabled. Otherwise, - the bytes are written unchanged, as before version 3.13.0. MessagePack - always writes the bytes as is, and BON8 always checks them (see - @ref check_utf8). + Used by @ref write_cbor, @ref write_msgpack, @ref write_ubjson (and so + @ref write_bjdata), and the BSON writing functions for string values and + object keys; never by @ref write_bon8, which always validates, since UTF-8 + lead bytes are structural there. - @param[in] s the string to check - @param[in] context the value that holds @a s (for diagnostics) - @throw type_error.316 if JSON_STRICT_BINARY_UTF8 is enabled and @a s is - not valid UTF-8 + - @ref error_handler_t::keep: @a s is returned unchanged, without even + checking it (the behavior of release 3.12.0 and earlier). + - @ref error_handler_t::strict: @ref check_utf8 is called, which throws + type_error.316 if @a s is not valid UTF-8. + - @ref error_handler_t::replace / @ref error_handler_t::ignore: @a s is + sanitized into @a storage with exactly the rules @ref + serializer::dump_escaped_impl uses, so that parsing what @ref + basic_json::dump produces for the same string and the same handler + yields the same result. + + Well-formed input is never copied: this returns a reference to @a s + itself in every case but a sanitized `replace`/`ignore` one, so @a + storage must outlive the returned reference only then. + + @param[in] s the string (value or object key) to write + @param[in] context the value @a s belongs to (for diagnostics) + @param[out] storage backing storage for a sanitized copy + + @return a reference to @a s, or to @a storage once it holds a sanitized copy */ - static void check_text_utf8(const string_t& s, const BasicJsonType& context) + const string_t& sanitize_utf8_for_write(const string_t& s, const BasicJsonType& context, string_t& storage) const { -#if JSON_STRICT_BINARY_UTF8 - check_utf8(s, context); -#else - static_cast(s); - static_cast(context); -#endif + switch (error_handler) + { + case error_handler_t::keep: + return s; + + case error_handler_t::strict: + check_utf8(s, context); + return s; + + case error_handler_t::replace: + case error_handler_t::ignore: + default: + if (is_valid_utf8(s)) + { + return s; + } + storage = sanitize_utf8(s, error_handler); + return storage; + } } /*! @@ -2521,6 +2606,10 @@ class binary_writer /// the output OutputSinkType oa; + + /// how to treat a string value or object key that is not valid UTF-8 + /// (CBOR, MessagePack, UBJSON, BJData, and BSON; not BON8) + const error_handler_t error_handler = binary_writer_default_error_handler(); }; } // namespace detail diff --git a/include/nlohmann/detail/output/error_handler.hpp b/include/nlohmann/detail/output/error_handler.hpp new file mode 100644 index 000000000..b95d70fce --- /dev/null +++ b/include/nlohmann/detail/output/error_handler.hpp @@ -0,0 +1,50 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#pragma once + +#include + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +/// how to treat decoding errors +/// +/// @ref basic_json::dump uses this to decide what to do with ill-formed +/// UTF-8 while escaping a string, and the binary writers (@ref +/// basic_json::to_cbor, @ref basic_json::to_ubjson, @ref +/// basic_json::to_bjdata, @ref basic_json::to_bson) use it the same way for +/// string values and object keys. The binary readers (@ref +/// basic_json::from_cbor, @ref basic_json::from_msgpack, @ref +/// basic_json::from_ubjson, @ref basic_json::from_bjdata, @ref +/// basic_json::from_bson) use it to decide whether to check text strings +/// and object keys for well-formed UTF-8 at all, since none of those +/// formats requires a decoder to do so. +enum class error_handler_t +{ + strict, ///< throw a type_error/parse_error exception in case of invalid UTF-8 + replace, ///< replace invalid UTF-8 sequences with U+FFFD + ignore, ///< ignore invalid UTF-8 sequences + keep ///< keep invalid UTF-8 sequences unchanged +}; + +/// the default error handler of the CBOR, UBJSON, BJData, and BSON writers: +/// error_handler_t::strict if JSON_STRICT_BINARY_UTF8 is enabled, otherwise +/// error_handler_t::keep (the behavior before version 3.13.0) +constexpr error_handler_t binary_writer_default_error_handler() noexcept +{ +#if JSON_STRICT_BINARY_UTF8 + return error_handler_t::strict; +#else + return error_handler_t::keep; +#endif +} + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END diff --git a/include/nlohmann/detail/output/serializer.hpp b/include/nlohmann/detail/output/serializer.hpp index 1c519d4c7..dcaae8055 100644 --- a/include/nlohmann/detail/output/serializer.hpp +++ b/include/nlohmann/detail/output/serializer.hpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -41,14 +42,6 @@ namespace detail // serialization // /////////////////// -/// how to treat decoding errors -enum class error_handler_t -{ - strict, ///< throw a type_error exception in case of invalid UTF-8 - replace, ///< replace invalid UTF-8 sequences with U+FFFD - ignore ///< ignore invalid UTF-8 sequences -}; - template class serializer { @@ -839,6 +832,16 @@ class serializer // EnsureAscii parameter is used, non-ASCII characters if ((codepoint <= 0x1F) || (EnsureAscii && (codepoint >= 0x7F))) { + if (EnsureAscii && error_handler == error_handler_t::keep) + { + // this character was buffered as raw bytes + // below in case it turned out to be part of + // an ill-formed sequence (which is kept as + // is); now that it decoded to a well-formed + // code point, undo that and \u-escape it + // like any other character instead + bytes = bytes_after_last_accept; + } if (codepoint <= 0xFFFF) { write_u_escape(bytes, static_cast(codepoint)); @@ -937,6 +940,44 @@ class serializer break; } + case error_handler_t::keep: + { + // the bytes of this (now abandoned) ill-formed + // sequence seen so far are already buffered below + // and are kept unchanged in the output + if (undumped_chars > 0) + { + // the byte that ended the sequence may be OK + // for itself (e.g., a quote that must still be + // escaped, or the lead byte of a well-formed + // code point), so read it again + --i; + } + else + { + // a byte that cannot start a sequence (e.g., + // 0xFF or a stray continuation byte) is kept + // as well + string_buffer[bytes++] = s[i]; + } + + // write buffer and reset index; there must be 13 bytes + // left, as this is the maximal number of bytes to be + // written ("\uxxxx\uxxxx\0") for one code point + if (string_buffer.size() - bytes < 13) + { + put_buffer(string_buffer, bytes); + bytes = 0; + } + + bytes_after_last_accept = bytes; + undumped_chars = 0; + + // continue processing the string + state = UTF8_ACCEPT; + break; + } + default: // LCOV_EXCL_LINE JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE } @@ -945,9 +986,12 @@ class serializer default: // decode found yet incomplete multibyte code point { - if (!EnsureAscii) + if (!EnsureAscii || error_handler == error_handler_t::keep) { - // code point will not be escaped - copy byte to buffer + // code point will not be escaped (or will be kept as + // is if it turns out to be ill-formed) - copy byte to + // buffer; dropped again above if it decodes to a + // well-formed code point that needs \u-escaping string_buffer[bytes++] = s[i]; } ++undumped_chars; @@ -998,6 +1042,14 @@ class serializer break; } + case error_handler_t::keep: + { + // write the ill-formed trailing bytes as is; they were + // buffered above regardless of EnsureAscii + put_buffer(string_buffer, bytes); + break; + } + default: // LCOV_EXCL_LINE JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE } diff --git a/include/nlohmann/detail/string_utils.hpp b/include/nlohmann/detail/string_utils.hpp index 2b6864d0d..4ef748f13 100644 --- a/include/nlohmann/detail/string_utils.hpp +++ b/include/nlohmann/detail/string_utils.hpp @@ -16,6 +16,7 @@ #include #include +#include NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail @@ -179,5 +180,134 @@ inline std::uint8_t decode(std::uint8_t& state, std::uint32_t& codep, const std: return state; } +/*! +@brief check a string for well-formed UTF-8 (RFC 3629, section 4) + +Used by the binary readers (CBOR, MessagePack, UBJSON, BJData, BSON) when an +@ref error_handler_t other than `keep` is requested for a text string value +or object key: none of those formats requires a decoder to reject ill-formed +UTF-8 on its own, so the check is opt-in there, unlike the JSON lexer and the +serializer's @ref decode -based escaping, which always run it. + +@param[in] s the string to check +@param[in] first the index to start checking at +@return whether `s.substr(first)` is well-formed UTF-8 + +@sa @ref decode +*/ +template +inline bool is_valid_utf8(const StringType& s, const std::size_t first = 0) noexcept +{ + std::uint8_t state = UTF8_ACCEPT; + std::uint32_t codepoint = 0; + + for (std::size_t i = first; i < s.size(); ++i) + { + decode(state, codepoint, static_cast(s[i])); + if (state == UTF8_REJECT) + { + return false; + } + } + + return state == UTF8_ACCEPT; +} + +/*! +@brief sanitize a string with ill-formed UTF-8 for @ref error_handler_t::replace or @ref error_handler_t::ignore + +Replaces every maximal ill-formed subsequence with U+FFFD (`replace`) or +drops it (`ignore`), using exactly the same boundaries @ref +serializer::dump_escaped_impl uses while escaping a string: a byte that does +not extend the sequence started by the previous byte(s) is reread as the +start of a new one, instead of being swallowed along with them. + +@pre @a error_handler is @ref error_handler_t::replace or @ref error_handler_t::ignore +@note Well-formed input is copied through unchanged, including bytes (e.g. + control characters or quotes) that @ref serializer::dump_escaped_impl + would itself escape; this function only concerns itself with + well-formedness, not with producing valid JSON text. + +@param[in] s the string to sanitize +@param[in] error_handler @ref error_handler_t::replace or @ref error_handler_t::ignore + +@return @a s with every ill-formed subsequence replaced or removed + +@sa @ref decode +*/ +template +inline StringType sanitize_utf8(const StringType& s, const error_handler_t error_handler) +{ + JSON_ASSERT(error_handler == error_handler_t::replace || error_handler == error_handler_t::ignore); + + StringType result; + result.reserve(s.size()); + + std::uint32_t codepoint = 0; + std::uint8_t state = UTF8_ACCEPT; + // length of result after the last accepted code point + std::size_t result_len_after_last_accept = 0; + // whether bytes of an as yet unresolved sequence were already appended + bool pending = false; + + for (std::size_t i = 0; i < s.size(); ++i) + { + switch (decode(state, codepoint, static_cast(s[i]))) + { + case UTF8_ACCEPT: // decode found a well-formed code point + { + result.push_back(s[i]); + result_len_after_last_accept = result.size(); + pending = false; + break; + } + + case UTF8_REJECT: // decode found an ill-formed byte + { + // in case we saw this byte for the first time, read it again, + // because it may be fine for itself, just not for the + // sequence that came before it + if (pending) + { + --i; + } + + // drop the bytes of the ill-formed sequence buffered below + result.resize(result_len_after_last_accept); + + if (error_handler == error_handler_t::replace) + { + result.append("\xEF\xBF\xBD"); + result_len_after_last_accept = result.size(); + } + + pending = false; + state = UTF8_ACCEPT; + break; + } + + default: // decode found yet incomplete multibyte code point + { + result.push_back(s[i]); + pending = true; + break; + } + } + } + + // the string ended with an incomplete sequence + if (state != UTF8_ACCEPT) + { + result.resize(result_len_after_last_accept); + + if (error_handler == error_handler_t::replace) + { + result.append("\xEF\xBF\xBD"); + } + } + + return result; +} + } // namespace detail NLOHMANN_JSON_NAMESPACE_END diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 07b625a89..421feda50 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -195,9 +195,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // used by the vector-returning to_* overloads template using vector_binary_writer = ::nlohmann::detail::binary_writer>; - template static vector_binary_writer vector_writer(std::vector& v) + template static vector_binary_writer vector_writer( + std::vector& v, const ::nlohmann::detail::error_handler_t error_handler = ::nlohmann::detail::binary_writer_default_error_handler()) { - return vector_binary_writer(::nlohmann::detail::output_vector_sink(v)); + return vector_binary_writer(::nlohmann::detail::output_vector_sink(v), error_handler); } JSON_PRIVATE_UNLESS_TESTED: @@ -5583,11 +5584,12 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec template static basic_json from_binary_impl(InputAdapterType ia, const input_format_t format, const bool strict, const bool allow_exceptions, + const error_handler_t error_handler = error_handler_t::keep, const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error) { basic_json result; detail::json_sax_dom_parser sdp(result, allow_exceptions); - binary_reader reader(std::move(ia), format); + binary_reader reader(std::move(ia), format, error_handler); if (!reader.sax_parse(&sdp, strict, tag_handler)) { result = value_t::discarded; @@ -5605,78 +5607,87 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec public: /// @brief create a CBOR serialization of a given JSON value /// @sa https://json.nlohmann.me/api/basic_json/to_cbor/ - static std::vector to_cbor(const basic_json& j) + static std::vector to_cbor(const basic_json& j, + const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { std::vector result; result.reserve(detail::binary_reserve_hint(j)); - vector_writer(result).write_cbor(j); + vector_writer(result, error_handler).write_cbor(j); return result; } /// @brief create a CBOR serialization of a given JSON value /// @sa https://json.nlohmann.me/api/basic_json/to_cbor/ - static void to_cbor(const basic_json& j, detail::output_adapter o) + static void to_cbor(const basic_json& j, detail::output_adapter o, + const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { - binary_writer(o).write_cbor(j); + binary_writer(o, error_handler).write_cbor(j); } /// @brief create a CBOR serialization of a given JSON value /// @sa https://json.nlohmann.me/api/basic_json/to_cbor/ - static void to_cbor(const basic_json& j, detail::output_adapter o) + static void to_cbor(const basic_json& j, detail::output_adapter o, + const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { - binary_writer(o).write_cbor(j); + binary_writer(o, error_handler).write_cbor(j); } /// @brief create a MessagePack serialization of a given JSON value /// @sa https://json.nlohmann.me/api/basic_json/to_msgpack/ - static std::vector to_msgpack(const basic_json& j) + static std::vector to_msgpack(const basic_json& j, + const error_handler_t error_handler = error_handler_t::keep) { std::vector result; result.reserve(detail::binary_reserve_hint(j)); - vector_writer(result).write_msgpack(j); + vector_writer(result, error_handler).write_msgpack(j); return result; } /// @brief create a MessagePack serialization of a given JSON value /// @sa https://json.nlohmann.me/api/basic_json/to_msgpack/ - static void to_msgpack(const basic_json& j, detail::output_adapter o) + static void to_msgpack(const basic_json& j, detail::output_adapter o, + const error_handler_t error_handler = error_handler_t::keep) { - binary_writer(o).write_msgpack(j); + binary_writer(o, error_handler).write_msgpack(j); } /// @brief create a MessagePack serialization of a given JSON value /// @sa https://json.nlohmann.me/api/basic_json/to_msgpack/ - static void to_msgpack(const basic_json& j, detail::output_adapter o) + static void to_msgpack(const basic_json& j, detail::output_adapter o, + const error_handler_t error_handler = error_handler_t::keep) { - binary_writer(o).write_msgpack(j); + binary_writer(o, error_handler).write_msgpack(j); } /// @brief create a UBJSON serialization of a given JSON value /// @sa https://json.nlohmann.me/api/basic_json/to_ubjson/ static std::vector to_ubjson(const basic_json& j, const bool use_size = false, - const bool use_type = false) + const bool use_type = false, + const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { std::vector result; result.reserve(detail::binary_reserve_hint(j)); - vector_writer(result).write_ubjson(j, use_size, use_type); + vector_writer(result, error_handler).write_ubjson(j, use_size, use_type); return result; } /// @brief create a UBJSON serialization of a given JSON value /// @sa https://json.nlohmann.me/api/basic_json/to_ubjson/ static void to_ubjson(const basic_json& j, detail::output_adapter o, - const bool use_size = false, const bool use_type = false) + const bool use_size = false, const bool use_type = false, + const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { - binary_writer(o).write_ubjson(j, use_size, use_type); + binary_writer(o, error_handler).write_ubjson(j, use_size, use_type); } /// @brief create a UBJSON serialization of a given JSON value /// @sa https://json.nlohmann.me/api/basic_json/to_ubjson/ static void to_ubjson(const basic_json& j, detail::output_adapter o, - const bool use_size = false, const bool use_type = false) + const bool use_size = false, const bool use_type = false, + const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { - binary_writer(o).write_ubjson(j, use_size, use_type); + binary_writer(o, error_handler).write_ubjson(j, use_size, use_type); } /// @brief create a BJData serialization of a given JSON value @@ -5684,11 +5695,12 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec static std::vector to_bjdata(const basic_json& j, const bool use_size = false, const bool use_type = false, - const bjdata_version_t version = bjdata_version_t::draft2) + const bjdata_version_t version = bjdata_version_t::draft2, + const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { std::vector result; result.reserve(detail::binary_reserve_hint(j)); - vector_writer(result).write_ubjson(j, use_size, use_type, true, true, version); + vector_writer(result, error_handler).write_ubjson(j, use_size, use_type, true, true, version); return result; } @@ -5696,42 +5708,47 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/to_bjdata/ static void to_bjdata(const basic_json& j, detail::output_adapter o, const bool use_size = false, const bool use_type = false, - const bjdata_version_t version = bjdata_version_t::draft2) + const bjdata_version_t version = bjdata_version_t::draft2, + const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { - binary_writer(o).write_ubjson(j, use_size, use_type, true, true, version); + binary_writer(o, error_handler).write_ubjson(j, use_size, use_type, true, true, version); } /// @brief create a BJData serialization of a given JSON value /// @sa https://json.nlohmann.me/api/basic_json/to_bjdata/ static void to_bjdata(const basic_json& j, detail::output_adapter o, const bool use_size = false, const bool use_type = false, - const bjdata_version_t version = bjdata_version_t::draft2) + const bjdata_version_t version = bjdata_version_t::draft2, + const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { - binary_writer(o).write_ubjson(j, use_size, use_type, true, true, version); + binary_writer(o, error_handler).write_ubjson(j, use_size, use_type, true, true, version); } /// @brief create a BSON serialization of a given JSON value /// @sa https://json.nlohmann.me/api/basic_json/to_bson/ - static std::vector to_bson(const basic_json& j) + static std::vector to_bson(const basic_json& j, + const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { std::vector result; result.reserve(detail::binary_reserve_hint(j)); - vector_writer(result).write_bson(j); + vector_writer(result, error_handler).write_bson(j); return result; } /// @brief create a BSON serialization of a given JSON value /// @sa https://json.nlohmann.me/api/basic_json/to_bson/ - static void to_bson(const basic_json& j, detail::output_adapter o) + static void to_bson(const basic_json& j, detail::output_adapter o, + const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { - binary_writer(o).write_bson(j); + binary_writer(o, error_handler).write_bson(j); } /// @brief create a BSON serialization of a given JSON value /// @sa https://json.nlohmann.me/api/basic_json/to_bson/ - static void to_bson(const basic_json& j, detail::output_adapter o) + static void to_bson(const basic_json& j, detail::output_adapter o, + const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { - binary_writer(o).write_bson(j); + binary_writer(o, error_handler).write_bson(j); } /// @brief create a BON8 serialization of a given JSON value @@ -5765,9 +5782,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec static basic_json from_cbor(InputType&& i, const bool strict = true, const bool allow_exceptions = true, - const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error) + const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error, + const error_handler_t error_handler = error_handler_t::keep) { - return from_binary_impl(detail::input_adapter(std::forward(i)), input_format_t::cbor, strict, allow_exceptions, tag_handler); + return from_binary_impl(detail::input_adapter(std::forward(i)), input_format_t::cbor, strict, allow_exceptions, error_handler, tag_handler); } /// @brief create a JSON value from an input in CBOR format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -5778,9 +5796,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec static basic_json from_cbor(IteratorType first, SentinelType last, const bool strict = true, const bool allow_exceptions = true, - const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error) + const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error, + const error_handler_t error_handler = error_handler_t::keep) { - return from_binary_impl(detail::input_adapter(std::move(first), std::move(last)), input_format_t::cbor, strict, allow_exceptions, tag_handler); + return from_binary_impl(detail::input_adapter(std::move(first), std::move(last)), input_format_t::cbor, strict, allow_exceptions, error_handler, tag_handler); } template @@ -5801,7 +5820,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool allow_exceptions = true, const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error) { - return from_binary_impl(i.get(), input_format_t::cbor, strict, allow_exceptions, tag_handler); + return from_binary_impl(i.get(), input_format_t::cbor, strict, allow_exceptions, error_handler_t::keep, tag_handler); } /// @brief create a JSON value from an input in MessagePack format @@ -5810,9 +5829,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec JSON_HEDLEY_WARN_UNUSED_RESULT static basic_json from_msgpack(InputType&& i, const bool strict = true, - const bool allow_exceptions = true) + const bool allow_exceptions = true, + const error_handler_t error_handler = error_handler_t::keep) { - return from_binary_impl(detail::input_adapter(std::forward(i)), input_format_t::msgpack, strict, allow_exceptions); + return from_binary_impl(detail::input_adapter(std::forward(i)), input_format_t::msgpack, strict, allow_exceptions, error_handler); } /// @brief create a JSON value from an input in MessagePack format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -5822,9 +5842,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec JSON_HEDLEY_WARN_UNUSED_RESULT static basic_json from_msgpack(IteratorType first, SentinelType last, const bool strict = true, - const bool allow_exceptions = true) + const bool allow_exceptions = true, + const error_handler_t error_handler = error_handler_t::keep) { - return from_binary_impl(detail::input_adapter(std::move(first), std::move(last)), input_format_t::msgpack, strict, allow_exceptions); + return from_binary_impl(detail::input_adapter(std::move(first), std::move(last)), input_format_t::msgpack, strict, allow_exceptions, error_handler); } template @@ -5852,9 +5873,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec JSON_HEDLEY_WARN_UNUSED_RESULT static basic_json from_ubjson(InputType&& i, const bool strict = true, - const bool allow_exceptions = true) + const bool allow_exceptions = true, + const error_handler_t error_handler = error_handler_t::keep) { - return from_binary_impl(detail::input_adapter(std::forward(i)), input_format_t::ubjson, strict, allow_exceptions); + return from_binary_impl(detail::input_adapter(std::forward(i)), input_format_t::ubjson, strict, allow_exceptions, error_handler); } /// @brief create a JSON value from an input in UBJSON format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -5864,9 +5886,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec JSON_HEDLEY_WARN_UNUSED_RESULT static basic_json from_ubjson(IteratorType first, SentinelType last, const bool strict = true, - const bool allow_exceptions = true) + const bool allow_exceptions = true, + const error_handler_t error_handler = error_handler_t::keep) { - return from_binary_impl(detail::input_adapter(std::move(first), std::move(last)), input_format_t::ubjson, strict, allow_exceptions); + return from_binary_impl(detail::input_adapter(std::move(first), std::move(last)), input_format_t::ubjson, strict, allow_exceptions, error_handler); } template @@ -5894,9 +5917,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec JSON_HEDLEY_WARN_UNUSED_RESULT static basic_json from_bjdata(InputType&& i, const bool strict = true, - const bool allow_exceptions = true) + const bool allow_exceptions = true, + const error_handler_t error_handler = error_handler_t::keep) { - return from_binary_impl(detail::input_adapter(std::forward(i)), input_format_t::bjdata, strict, allow_exceptions); + return from_binary_impl(detail::input_adapter(std::forward(i)), input_format_t::bjdata, strict, allow_exceptions, error_handler); } /// @brief create a JSON value from an input in BJData format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -5906,9 +5930,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec JSON_HEDLEY_WARN_UNUSED_RESULT static basic_json from_bjdata(IteratorType first, SentinelType last, const bool strict = true, - const bool allow_exceptions = true) + const bool allow_exceptions = true, + const error_handler_t error_handler = error_handler_t::keep) { - return from_binary_impl(detail::input_adapter(std::move(first), std::move(last)), input_format_t::bjdata, strict, allow_exceptions); + return from_binary_impl(detail::input_adapter(std::move(first), std::move(last)), input_format_t::bjdata, strict, allow_exceptions, error_handler); } /// @brief create a JSON value from an input in BON8 format @@ -5940,9 +5965,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec JSON_HEDLEY_WARN_UNUSED_RESULT static basic_json from_bson(InputType&& i, const bool strict = true, - const bool allow_exceptions = true) + const bool allow_exceptions = true, + const error_handler_t error_handler = error_handler_t::keep) { - return from_binary_impl(detail::input_adapter(std::forward(i)), input_format_t::bson, strict, allow_exceptions); + return from_binary_impl(detail::input_adapter(std::forward(i)), input_format_t::bson, strict, allow_exceptions, error_handler); } /// @brief create a JSON value from an input in BSON format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -5952,9 +5978,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec JSON_HEDLEY_WARN_UNUSED_RESULT static basic_json from_bson(IteratorType first, SentinelType last, const bool strict = true, - const bool allow_exceptions = true) + const bool allow_exceptions = true, + const error_handler_t error_handler = error_handler_t::keep) { - return from_binary_impl(detail::input_adapter(std::move(first), std::move(last)), input_format_t::bson, strict, allow_exceptions); + return from_binary_impl(detail::input_adapter(std::move(first), std::move(last)), input_format_t::bson, strict, allow_exceptions, error_handler); } template diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 16e87cc86..511401699 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -104,6 +104,10 @@ #define JSON_STRICT_NUL_HANDLING 0 #endif +#ifndef JSON_STRICT_BINARY_UTF8 + #define JSON_STRICT_BINARY_UTF8 0 +#endif + #if JSON_DIAGNOSTICS #define NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS _diag #else @@ -140,14 +144,20 @@ #define NLOHMANN_JSON_ABI_TAG_STRICT_NUL_HANDLING #endif +#if JSON_STRICT_BINARY_UTF8 + #define NLOHMANN_JSON_ABI_TAG_STRICT_BINARY_UTF8 _sbu8 +#else + #define NLOHMANN_JSON_ABI_TAG_STRICT_BINARY_UTF8 +#endif + #ifndef NLOHMANN_JSON_NAMESPACE_NO_VERSION #define NLOHMANN_JSON_NAMESPACE_NO_VERSION 0 #endif // Construct the namespace ABI tags component -#define NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b, c, d, e, f) json_abi ## a ## b ## c ## d ## e ## f -#define NLOHMANN_JSON_ABI_TAGS_CONCAT(a, b, c, d, e, f) \ - NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b, c, d, e, f) +#define NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b, c, d, e, f, g) json_abi ## a ## b ## c ## d ## e ## f ## g +#define NLOHMANN_JSON_ABI_TAGS_CONCAT(a, b, c, d, e, f, g) \ + NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b, c, d, e, f, g) #define NLOHMANN_JSON_ABI_TAGS \ NLOHMANN_JSON_ABI_TAGS_CONCAT( \ @@ -156,7 +166,8 @@ NLOHMANN_JSON_ABI_TAG_DIAGNOSTIC_POSITIONS, \ NLOHMANN_JSON_ABI_TAG_BRACE_INIT_COPY_SEMANTICS, \ NLOHMANN_JSON_ABI_TAG_PRECISE_STREAM_POSITION, \ - NLOHMANN_JSON_ABI_TAG_STRICT_NUL_HANDLING) + NLOHMANN_JSON_ABI_TAG_STRICT_NUL_HANDLING, \ + NLOHMANN_JSON_ABI_TAG_STRICT_BINARY_UTF8) // Construct the namespace version component #define NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT_EX(major, minor, patch) \ @@ -6151,6 +6162,59 @@ NLOHMANN_JSON_NAMESPACE_END // #include +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +// #include + + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +/// how to treat decoding errors +/// +/// @ref basic_json::dump uses this to decide what to do with ill-formed +/// UTF-8 while escaping a string, and the binary writers (@ref +/// basic_json::to_cbor, @ref basic_json::to_ubjson, @ref +/// basic_json::to_bjdata, @ref basic_json::to_bson) use it the same way for +/// string values and object keys. The binary readers (@ref +/// basic_json::from_cbor, @ref basic_json::from_msgpack, @ref +/// basic_json::from_ubjson, @ref basic_json::from_bjdata, @ref +/// basic_json::from_bson) use it to decide whether to check text strings +/// and object keys for well-formed UTF-8 at all, since none of those +/// formats requires a decoder to do so. +enum class error_handler_t +{ + strict, ///< throw a type_error/parse_error exception in case of invalid UTF-8 + replace, ///< replace invalid UTF-8 sequences with U+FFFD + ignore, ///< ignore invalid UTF-8 sequences + keep ///< keep invalid UTF-8 sequences unchanged +}; + +/// the default error handler of the CBOR, UBJSON, BJData, and BSON writers: +/// error_handler_t::strict if JSON_STRICT_BINARY_UTF8 is enabled, otherwise +/// error_handler_t::keep (the behavior before version 3.13.0) +constexpr error_handler_t binary_writer_default_error_handler() noexcept +{ +#if JSON_STRICT_BINARY_UTF8 + return error_handler_t::strict; +#else + return error_handler_t::keep; +#endif +} + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail @@ -6252,13 +6316,14 @@ This is a single-byte step of a "shift-based" UTF-8 decoder originally written by Björn Hoehrmann. See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details. -The library checks UTF-8 well-formedness (RFC 3629, section 4) in four +The library checks UTF-8 well-formedness (RFC 3629, section 4) in three places, which differ in speed, diagnostics, and how they read the input: -- decode() and @ref is_valid_utf8 below: the serializer (to escape and, in - strict mode, reject ill-formed UTF-8 when dumping a string) and the CBOR, - MessagePack, BSON, UBJSON and BJData readers (to reject ill-formed UTF-8 in - text strings at decode time). +- decode() below: the serializer, to escape and, in strict mode, reject + ill-formed UTF-8 when dumping a string. The CBOR, MessagePack, BSON, + UBJSON and BJData readers do not use it: none of those specs requires a + decoder to reject ill-formed UTF-8 in text strings, so the readers keep + the bytes as is and leave the check to dump() and the binary writers. - the per-lead-byte switch in lexer::scan_string(): JSON text, with a diagnostic for each kind of error. - validate_one_utf8() and valid_utf8_prefix() in string_scan.hpp: the lexer's @@ -6314,19 +6379,19 @@ inline std::uint8_t decode(std::uint8_t& state, std::uint32_t& codep, const std: } /*! -@brief check whether a string consists solely of valid UTF-8 +@brief check a string for well-formed UTF-8 (RFC 3629, section 4) -Used by the CBOR/MessagePack/BSON/UBJSON binary readers to reject text -strings that are not valid UTF-8 at decode time (RFC 8949 §3.1 and the -MessagePack/BSON specifications all require text strings to be UTF-8), so -that malformed input is caught immediately instead of only surfacing later -as a type_error.316 when the resulting value is dumped. +Used by the binary readers (CBOR, MessagePack, UBJSON, BJData, BSON) when an +@ref error_handler_t other than `keep` is requested for a text string value +or object key: none of those formats requires a decoder to reject ill-formed +UTF-8 on its own, so the check is opt-in there, unlike the JSON lexer and the +serializer's @ref decode -based escaping, which always run it. @param[in] s the string to check -@param[in] first index of the first byte to check; the bytes before it are - assumed to have been validated already and to end on a - code point boundary -@return whether @a s (from index @a first on) is valid UTF-8 +@param[in] first the index to start checking at +@return whether `s.substr(first)` is well-formed UTF-8 + +@sa @ref decode */ template inline bool is_valid_utf8(const StringType& s, const std::size_t first = 0) noexcept @@ -6346,6 +6411,102 @@ inline bool is_valid_utf8(const StringType& s, const std::size_t first = 0) noex return state == UTF8_ACCEPT; } +/*! +@brief sanitize a string with ill-formed UTF-8 for @ref error_handler_t::replace or @ref error_handler_t::ignore + +Replaces every maximal ill-formed subsequence with U+FFFD (`replace`) or +drops it (`ignore`), using exactly the same boundaries @ref +serializer::dump_escaped_impl uses while escaping a string: a byte that does +not extend the sequence started by the previous byte(s) is reread as the +start of a new one, instead of being swallowed along with them. + +@pre @a error_handler is @ref error_handler_t::replace or @ref error_handler_t::ignore +@note Well-formed input is copied through unchanged, including bytes (e.g. + control characters or quotes) that @ref serializer::dump_escaped_impl + would itself escape; this function only concerns itself with + well-formedness, not with producing valid JSON text. + +@param[in] s the string to sanitize +@param[in] error_handler @ref error_handler_t::replace or @ref error_handler_t::ignore + +@return @a s with every ill-formed subsequence replaced or removed + +@sa @ref decode +*/ +template +inline StringType sanitize_utf8(const StringType& s, const error_handler_t error_handler) +{ + JSON_ASSERT(error_handler == error_handler_t::replace || error_handler == error_handler_t::ignore); + + StringType result; + result.reserve(s.size()); + + std::uint32_t codepoint = 0; + std::uint8_t state = UTF8_ACCEPT; + // length of result after the last accepted code point + std::size_t result_len_after_last_accept = 0; + // whether bytes of an as yet unresolved sequence were already appended + bool pending = false; + + for (std::size_t i = 0; i < s.size(); ++i) + { + switch (decode(state, codepoint, static_cast(s[i]))) + { + case UTF8_ACCEPT: // decode found a well-formed code point + { + result.push_back(s[i]); + result_len_after_last_accept = result.size(); + pending = false; + break; + } + + case UTF8_REJECT: // decode found an ill-formed byte + { + // in case we saw this byte for the first time, read it again, + // because it may be fine for itself, just not for the + // sequence that came before it + if (pending) + { + --i; + } + + // drop the bytes of the ill-formed sequence buffered below + result.resize(result_len_after_last_accept); + + if (error_handler == error_handler_t::replace) + { + result.append("\xEF\xBF\xBD"); + result_len_after_last_accept = result.size(); + } + + pending = false; + state = UTF8_ACCEPT; + break; + } + + default: // decode found yet incomplete multibyte code point + { + result.push_back(s[i]); + pending = true; + break; + } + } + } + + // the string ended with an incomplete sequence + if (state != UTF8_ACCEPT) + { + result.resize(result_len_after_last_accept); + + if (error_handler == error_handler_t::replace) + { + result.append("\xEF\xBF\xBD"); + } + } + + return result; +} + } // namespace detail NLOHMANN_JSON_NAMESPACE_END @@ -13452,6 +13613,8 @@ NLOHMANN_JSON_NAMESPACE_END // #include +// #include + // #include // #include @@ -13532,8 +13695,16 @@ class binary_reader @brief create a binary reader @param[in] adapter input adapter to read from + @param[in] format the binary format to parse + @param[in] error_handler how to treat text strings and object keys that + are not well-formed UTF-8; none of the supported formats + requires a decoder to reject those, so the default is to + @ref error_handler_t::keep them unchanged, as every binary + reader did before this parameter existed */ - explicit binary_reader(InputAdapterType&& adapter, const input_format_t format = input_format_t::json) noexcept : ia(std::move(adapter)), input_format(format) + explicit binary_reader(InputAdapterType&& adapter, const input_format_t format = input_format_t::json, + const error_handler_t error_handler = error_handler_t::keep) noexcept + : ia(std::move(adapter)), input_format(format), error_handler(error_handler) { (void)detail::is_sax_static_asserts {}; } @@ -13852,7 +14023,7 @@ class binary_reader { if (get_bson_cstr_bulk(result, std::integral_constant {})) { - return true; + return check_string_utf8(result, "key"); } auto out = std::back_inserter(result); @@ -13865,7 +14036,7 @@ class binary_reader } if (current == 0x00) { - return true; + return check_string_utf8(result, "key"); } *out++ = static_cast(current); } @@ -13946,7 +14117,7 @@ class binary_reader "string"), nullptr)); } - return true; + return check_string_utf8(result, "string"); } /*! @@ -14573,7 +14744,7 @@ class binary_reader @return whether string creation completed */ - bool get_cbor_string(string_t& result) + bool get_cbor_string(string_t& result, const char* context = "string") { // number of indefinite-length strings that have been opened and not // closed yet. RFC 8949, Section 3.2.3 does not permit nesting them, @@ -14603,7 +14774,7 @@ class binary_reader { if (--open == 0) { - return true; + return check_string_utf8(result, context); } get(); continue; @@ -14616,7 +14787,7 @@ class binary_reader if (open == 0) { - return true; + return check_string_utf8(result, context); } get(); @@ -14640,7 +14811,7 @@ class binary_reader // EOF and major type 3 (text string) are left to get_cbor_string if (current == char_traits::eof() || (static_cast(current) & 0xE0u) == 0x60u) { - return get_cbor_string(result); + return get_cbor_string(result, "key"); } const char* found = nullptr; @@ -15428,7 +15599,7 @@ class binary_reader @return whether string creation completed */ - bool get_msgpack_string(string_t& result) + bool get_msgpack_string(string_t& result, const char* context = "string") { if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::msgpack, "string"))) { @@ -15471,25 +15642,25 @@ class binary_reader case 0xBE: case 0xBF: { - return get_string(input_format_t::msgpack, static_cast(current) & 0x1Fu, result); + return get_string(input_format_t::msgpack, static_cast(current) & 0x1Fu, result) && check_string_utf8(result, context); } case 0xD9: // str 8 { std::uint8_t len{}; - return get_number(input_format_t::msgpack, len) && get_string(input_format_t::msgpack, len, result); + return get_number(input_format_t::msgpack, len) && get_string(input_format_t::msgpack, len, result) && check_string_utf8(result, context); } case 0xDA: // str 16 { std::uint16_t len{}; - return get_number(input_format_t::msgpack, len) && get_string(input_format_t::msgpack, len, result); + return get_number(input_format_t::msgpack, len) && get_string(input_format_t::msgpack, len, result) && check_string_utf8(result, context); } case 0xDB: // str 32 { std::uint32_t len{}; - return get_number(input_format_t::msgpack, len) && get_string(input_format_t::msgpack, len, result); + return get_number(input_format_t::msgpack, len) && get_string(input_format_t::msgpack, len, result) && check_string_utf8(result, context); } default: @@ -15567,7 +15738,7 @@ class binary_reader // byte 0xC1 are left to get_msgpack_string if (current == char_traits::eof()) { - return get_msgpack_string(result); + return get_msgpack_string(result, "key"); } if (current <= 0x7F || current >= 0xE0) { @@ -15583,7 +15754,7 @@ class binary_reader } else { - return get_msgpack_string(result); + return get_msgpack_string(result, "key"); } break; } @@ -15829,7 +16000,7 @@ class binary_reader if (top.is_object) { key.clear(); - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key))) + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key, true, "key") || !sax->key(key))) { return false; } @@ -15851,7 +16022,7 @@ class binary_reader if (top.is_object) { key.clear(); - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key, false) || !sax->key(key))) + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key, false, "key") || !sax->key(key))) { return false; } @@ -15919,7 +16090,7 @@ class binary_reader @return whether string creation completed */ - bool get_ubjson_string(string_t& result, const bool get_char = true) + bool get_ubjson_string(string_t& result, const bool get_char = true, const char* context = "string") { if (get_char) { @@ -15940,31 +16111,31 @@ class binary_reader case 'U': { std::uint8_t len{}; - return get_number(input_format, len) && get_string(input_format, len, result); + return get_number(input_format, len) && get_string(input_format, len, result) && check_string_utf8(result, context); } case 'i': { std::int8_t len{}; - return get_number(input_format, len) && check_ubjson_string_length(len) && get_string(input_format, len, result); + return get_number(input_format, len) && check_ubjson_string_length(len) && get_string(input_format, len, result) && check_string_utf8(result, context); } case 'I': { std::int16_t len{}; - return get_number(input_format, len) && check_ubjson_string_length(len) && get_string(input_format, len, result); + return get_number(input_format, len) && check_ubjson_string_length(len) && get_string(input_format, len, result) && check_string_utf8(result, context); } case 'l': { std::int32_t len{}; - return get_number(input_format, len) && check_ubjson_string_length(len) && get_string(input_format, len, result); + return get_number(input_format, len) && check_ubjson_string_length(len) && get_string(input_format, len, result) && check_string_utf8(result, context); } case 'L': { std::int64_t len{}; - return get_number(input_format, len) && check_ubjson_string_length(len) && get_string(input_format, len, result); + return get_number(input_format, len) && check_ubjson_string_length(len) && get_string(input_format, len, result) && check_string_utf8(result, context); } case 'u': @@ -15974,7 +16145,7 @@ class binary_reader break; } std::uint16_t len{}; - return get_number(input_format, len) && get_string(input_format, len, result); + return get_number(input_format, len) && get_string(input_format, len, result) && check_string_utf8(result, context); } case 'm': @@ -15984,7 +16155,7 @@ class binary_reader break; } std::uint32_t len{}; - return get_number(input_format, len) && get_string(input_format, len, result); + return get_number(input_format, len) && get_string(input_format, len, result) && check_string_utf8(result, context); } case 'M': @@ -15994,7 +16165,7 @@ class binary_reader break; } std::uint64_t len{}; - return get_number(input_format, len) && get_string(input_format, len, result); + return get_number(input_format, len) && get_string(input_format, len, result) && check_string_utf8(result, context); } default: @@ -17468,27 +17639,50 @@ class binary_reader const NumberType len, string_t& result) { - // get_bytes() appends to result, and CBOR indefinite-length strings - // collect all their chunks in the same result; validating only the - // newly read bytes keeps the check linear in the input size - const std::size_t old_size = result.size(); - if (JSON_HEDLEY_UNLIKELY(!get_bytes(format, len, "string", result))) + // Strings are taken as is by default: none of CBOR (RFC 8949 §3.1 + // leaves the choice to the decoder), MessagePack (whose spec + // explicitly allows a str object to contain an invalid byte + // sequence), UBJSON, BJData, or BSON requires a decoder to reject + // ill-formed UTF-8. Checking (and, with @ref error_handler_t::strict, + // rejecting, or with `replace`/`ignore`, sanitizing) is opt-in via + // @ref error_handler, applied once the whole string (all chunks of + // an indefinite-length CBOR string included) has been assembled, by + // @ref check_string_utf8 at the call site. + return get_bytes(format, len, "string", result); + } + + /*! + @brief validate a decoded text string (value or object key) against @ref error_handler + + None of the binary formats requires a decoder to reject ill-formed UTF-8 + in a text string (see @ref get_string), so by default + (@ref error_handler_t::keep) this does nothing. A stricter + @ref error_handler opts into the same well-formedness check @ref + serializer::dump_escaped_impl applies when dumping a string: + @ref error_handler_t::strict rejects ill-formed input with + parse_error.113 (honoring `allow_exceptions` via @a sax), while + @ref error_handler_t::replace / @ref error_handler_t::ignore sanitize + @a result in place, using the exact same rules. + + @param[in,out] result the already assembled string to check + @param[in] context further context information (for diagnostics) + @return whether @a result is acceptable (always true for `keep`) + */ + bool check_string_utf8(string_t& result, const char* context) + { + if (error_handler == error_handler_t::keep || is_valid_utf8(result)) { - return false; + return true; } - // RFC 8949 (CBOR) §3.1 and the MessagePack/BSON/UBJSON specifications - // all require text strings to be valid UTF-8; reject anything else - // right here so malformed input is caught at decode time instead of - // only surfacing later as a type_error.316 when the value is dumped - // (which would defeat allow_exceptions=false / strict discarding). - if (JSON_HEDLEY_UNLIKELY(!is_valid_utf8(result, old_size))) + if (error_handler == error_handler_t::strict) { - return sax->parse_error(chars_read, get_token_string(), - parse_error::create(113, chars_read, - exception_message(format, "invalid string: ill-formed UTF-8 byte", "string"), nullptr)); + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, + exception_message(input_format, "invalid string: ill-formed UTF-8 byte", context), nullptr)); } + result = sanitize_utf8(result, error_handler); return true; } @@ -17665,6 +17859,9 @@ class binary_reader /// input format const input_format_t input_format = input_format_t::json; + /// how to treat text strings/object keys that are not well-formed UTF-8 + const error_handler_t error_handler = error_handler_t::keep; + /// the SAX parser json_sax_t* sax = nullptr; @@ -20753,6 +20950,8 @@ NLOHMANN_JSON_NAMESPACE_END // #include +// #include + // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ @@ -21120,8 +21319,12 @@ class binary_writer @param[in] sink output sink to write to (a value-type sink such as output_vector_sink, or output_adapter_sink wrapping a type-erased output adapter) + @param[in] error_handler_ how to treat a string value or object key that + is not valid UTF-8 (CBOR, MessagePack, UBJSON, BJData, and BSON; + never consulted by @ref write_bon8) */ - explicit binary_writer(OutputSinkType sink) : oa(std::move(sink)) + explicit binary_writer(OutputSinkType sink, const error_handler_t error_handler_ = binary_writer_default_error_handler()) + : oa(std::move(sink)), error_handler(error_handler_) {} /*! @@ -21134,14 +21337,20 @@ class binary_writer from one. @param[in] adapter output adapter to write to + @param[in] error_handler_ how to treat a string value or object key that + is not valid UTF-8 (CBOR, MessagePack, UBJSON, BJData, and BSON; + never consulted by @ref write_bon8) */ template < typename SinkType = OutputSinkType, typename std::enable_if < std::is_constructible>::value, int >::type = 0 > - explicit binary_writer(output_adapter_t adapter) : oa(SinkType(std::move(adapter))) + explicit binary_writer(output_adapter_t adapter, const error_handler_t error_handler_ = binary_writer_default_error_handler()) + : oa(SinkType(std::move(adapter))), error_handler(error_handler_) {} /*! @param[in] j JSON value to serialize + @throw type_error.316 if a string value or an object key is not valid + UTF-8 @throw type_error.317 if @a j is not an object */ void write_bson(const BasicJsonType& j) @@ -21172,6 +21381,8 @@ class binary_writer /*! @param[in] j JSON value to serialize + @throw type_error.316 if a string value or an object key is not valid + UTF-8 */ void write_cbor(const BasicJsonType& j) { @@ -21238,13 +21449,16 @@ class binary_writer case value_t::string: { + string_t storage; + const string_t& value = sanitize_utf8_for_write(*j.m_data.m_value.string, j, storage); + // step 1: write control byte and the string length - write_cbor_head(0x60, j.m_data.m_value.string->size()); + write_cbor_head(0x60, value.size()); // step 2: write the string oa.write_characters( - reinterpret_cast(j.m_data.m_value.string->data()), - j.m_data.m_value.string->size()); + reinterpret_cast(value.data()), + value.size()); break; } @@ -21314,6 +21528,17 @@ class binary_writer // step 2: write each element for (const auto& el : *j.m_data.m_value.object) { + // el.first is checked here, against the object as + // diagnostics context, because write_cbor(el.first) + // converts it to a temporary basic_json that would be + // used as the context instead; for error_handler_t::keep + // and ::replace/::ignore the recursive write_cbor(el.first) + // call below handles the key like any other string, so no + // separate check is needed here for those + if (error_handler == error_handler_t::strict) + { + check_utf8(el.first, j); + } write_cbor(el.first); write_cbor(el.second); } @@ -21461,8 +21686,11 @@ class binary_writer case value_t::string: { + string_t storage; + const string_t& value = sanitize_utf8_for_write(*j.m_data.m_value.string, j, storage); + // step 1: write control byte and the string length - const auto N = to_msgpack_length(j.m_data.m_value.string->size(), j); + const auto N = to_msgpack_length(value.size(), j); if (N <= 31) { // fixstr @@ -21489,8 +21717,8 @@ class binary_writer // step 2: write the string oa.write_characters( - reinterpret_cast(j.m_data.m_value.string->data()), - j.m_data.m_value.string->size()); + reinterpret_cast(value.data()), + value.size()); break; } @@ -21637,6 +21865,13 @@ class binary_writer // step 2: write each element for (const auto& el : *j.m_data.m_value.object) { + // as in write_cbor, el.first is checked here against the + // object as diagnostics context; the recursive call below + // handles keep/replace/ignore like any other string + if (error_handler == error_handler_t::strict) + { + check_utf8(el.first, j); + } write_msgpack(el.first); write_msgpack(el.second); } @@ -21656,6 +21891,8 @@ class binary_writer @param[in] add_prefix whether prefixes need to be used for this value @param[in] use_bjdata whether write in BJData format, default is false @param[in] bjdata_version which BJData version to use, default is draft2 + @throw type_error.316 if a string value or an object key is not valid + UTF-8 */ void write_ubjson(const BasicJsonType& j, const bool use_count, const bool use_type, const bool add_prefix = true, @@ -21705,14 +21942,17 @@ class binary_writer case value_t::string: { + string_t storage; + const string_t& value = sanitize_utf8_for_write(*j.m_data.m_value.string, j, storage); + if (add_prefix) { oa.write_character(to_char_type('S')); } - write_number_with_ubjson_prefix(j.m_data.m_value.string->size(), true, use_bjdata); + write_number_with_ubjson_prefix(value.size(), true, use_bjdata); oa.write_characters( - reinterpret_cast(j.m_data.m_value.string->data()), - j.m_data.m_value.string->size()); + reinterpret_cast(value.data()), + value.size()); break; } @@ -21867,10 +22107,12 @@ class binary_writer for (const auto& el : *j.m_data.m_value.object) { - write_number_with_ubjson_prefix(el.first.size(), true, use_bjdata); + string_t storage; + const string_t& key = sanitize_utf8_for_write(el.first, j, storage); + write_number_with_ubjson_prefix(key.size(), true, use_bjdata); oa.write_characters( - reinterpret_cast(el.first.data()), - el.first.size()); + reinterpret_cast(key.data()), + key.size()); write_ubjson(el.second, use_count, use_type, prefix_required, use_bjdata, bjdata_version); } @@ -21911,8 +22153,12 @@ class binary_writer /*! @return The size of a BSON document entry header, including the id marker and the entry name size (and its null-terminator). + @throw out_of_range.409 if @a name contains U+0000, before anything is + written + @throw type_error.316 if @a name is not valid UTF-8, before anything is + written */ - static std::size_t calc_bson_entry_header_size(const string_t& name, const BasicJsonType& j) + std::size_t calc_bson_entry_header_size(const string_t& name, const BasicJsonType& j) { const auto it = name.find(static_cast(0)); if (JSON_HEDLEY_UNLIKELY(it != BasicJsonType::string_t::npos)) @@ -21920,8 +22166,10 @@ class binary_writer JSON_THROW(out_of_range::create(409, concat("BSON key cannot contain code point U+0000 (at byte ", std::to_string(it), ")"), &j)); } - static_cast(j); - return /*id*/ 1ul + name.size() + /*zero-terminator*/1u; + string_t storage; + const string_t& sanitized = sanitize_utf8_for_write(name, j, storage); + + return /*id*/ 1ul + sanitized.size() + /*zero-terminator*/1u; } /*! @@ -21941,14 +22189,28 @@ class binary_writer /*! @brief Writes the given @a element_type and @a name to the output adapter + + @a name has already been validated (and, for @ref error_handler_t::strict, + found well-formed) by @ref calc_bson_entry_header_size during the earlier + size pass, so only @ref error_handler_t::replace / @ref + error_handler_t::ignore need to sanitize it again here, to actually write + the bytes that size was computed from. */ void write_bson_entry_header(const string_t& name, const std::uint8_t element_type) { oa.write_character(to_char_type(element_type)); - oa.write_characters( - reinterpret_cast(name.data()), - name.size()); + + if (error_handler == error_handler_t::keep || error_handler == error_handler_t::strict || is_valid_utf8(name)) + { + oa.write_characters(reinterpret_cast(name.data()), name.size()); + } + else + { + const string_t sanitized = sanitize_utf8(name, error_handler); + oa.write_characters(reinterpret_cast(sanitized.data()), sanitized.size()); + } + // the terminating null byte is written explicitly rather than taken // from the buffer, so that string_t::data() need not be null-terminated oa.write_character(to_char_type(0x00)); @@ -21976,24 +22238,50 @@ class binary_writer /*! @return The size of the BSON-encoded string in @a value + @throw type_error.316 if @a value is not valid UTF-8, before anything is + written + + @note The UTF-8 check is skipped if @a value is already too long for the + 32-bit BSON length field (@ref to_bson_length rejects it later, once + the size of the whole document is known); this also keeps the check + from reading past a StringType that reports a size larger than what + it actually holds. */ - static std::size_t calc_bson_string_size(const string_t& value) + std::size_t calc_bson_string_size(const string_t& value, const BasicJsonType& j) { + if (JSON_HEDLEY_LIKELY(value_in_range_of(value.size()))) + { + string_t storage; + const string_t& sanitized = sanitize_utf8_for_write(value, j, storage); + return sizeof(std::int32_t) + sanitized.size() + 1ul; + } return sizeof(std::int32_t) + value.size() + 1ul; } /*! @brief Writes a BSON element with key @a name and string value @a value + + @a value has already been validated (and, for @ref error_handler_t::strict, + found well-formed) by @ref calc_bson_string_size during the earlier size + pass, so only @ref error_handler_t::replace / @ref error_handler_t::ignore + need to sanitize it again here, to actually write the bytes that size was + computed from. */ void write_bson_string(const string_t& name, const string_t& value) { write_bson_entry_header(name, 0x02); - write_number(to_bson_length(value.size() + 1ul), true); + const bool sanitize = error_handler != error_handler_t::keep + && error_handler != error_handler_t::strict + && !is_valid_utf8(value); + const string_t sanitized = sanitize ? sanitize_utf8(value, error_handler) : string_t{}; + const string_t& written = sanitize ? sanitized : value; + + write_number(to_bson_length(written.size() + 1ul), true); oa.write_characters( - reinterpret_cast(value.data()), - value.size()); + reinterpret_cast(written.data()), + written.size()); // the terminating null byte is written explicitly rather than taken // from the buffer, so that string_t::data() need not be null-terminated oa.write_character(to_char_type(0x00)); @@ -22107,8 +22395,10 @@ class binary_writer is neither an object nor an array @throw out_of_range.415 if @a j is binary with a subtype that does not fit into a byte, before anything is written + @throw type_error.316 if @a j is a string that is not valid UTF-8, before + anything is written */ - static std::size_t calc_bson_value_size(const BasicJsonType& j) + std::size_t calc_bson_value_size(const BasicJsonType& j) { switch (j.type()) { @@ -22128,7 +22418,7 @@ class binary_writer return calc_bson_unsigned_size(j.m_data.m_value.number_unsigned); case value_t::string: - return calc_bson_string_size(*j.m_data.m_value.string); + return calc_bson_string_size(*j.m_data.m_value.string, j); case value_t::null: return 0ul; @@ -22241,8 +22531,10 @@ class binary_writer written @throw out_of_range.415 if a binary value's subtype does not fit into a byte, before anything is written + @throw type_error.316 if a string value or a key is not valid UTF-8, + before anything is written */ - static std::size_t calc_bson_sizes(const BasicJsonType& document, std::vector& nested_sizes) + std::size_t calc_bson_sizes(const BasicJsonType& document, std::vector& nested_sizes) { // the object or array whose entries are being sized, and the ones it // is in; nothing is allocated unless the document nests @@ -23119,7 +23411,7 @@ class binary_writer */ void write_bon8_string(const string_t& s, bool& string_open, const BasicJsonType& context) { - check_bon8_utf8(s, context); + check_utf8(s, context); // a string that follows another string terminates it if (string_open) @@ -23149,7 +23441,7 @@ class binary_writer @throw type_error.316 if @a s is not valid UTF-8; the message names the first byte of the first invalid or incomplete sequence */ - static void check_bon8_utf8(const string_t& s, const BasicJsonType& context) + static void check_utf8(const string_t& s, const BasicJsonType& context) { static_cast(context); // only used when exceptions are enabled const auto* data = reinterpret_cast(s.data()); @@ -23160,6 +23452,57 @@ class binary_writer } } + /*! + @brief return @a s as it should be written, honoring @ref error_handler + + Used by @ref write_cbor, @ref write_msgpack, @ref write_ubjson (and so + @ref write_bjdata), and the BSON writing functions for string values and + object keys; never by @ref write_bon8, which always validates, since UTF-8 + lead bytes are structural there. + + - @ref error_handler_t::keep: @a s is returned unchanged, without even + checking it (the behavior of release 3.12.0 and earlier). + - @ref error_handler_t::strict: @ref check_utf8 is called, which throws + type_error.316 if @a s is not valid UTF-8. + - @ref error_handler_t::replace / @ref error_handler_t::ignore: @a s is + sanitized into @a storage with exactly the rules @ref + serializer::dump_escaped_impl uses, so that parsing what @ref + basic_json::dump produces for the same string and the same handler + yields the same result. + + Well-formed input is never copied: this returns a reference to @a s + itself in every case but a sanitized `replace`/`ignore` one, so @a + storage must outlive the returned reference only then. + + @param[in] s the string (value or object key) to write + @param[in] context the value @a s belongs to (for diagnostics) + @param[out] storage backing storage for a sanitized copy + + @return a reference to @a s, or to @a storage once it holds a sanitized copy + */ + const string_t& sanitize_utf8_for_write(const string_t& s, const BasicJsonType& context, string_t& storage) const + { + switch (error_handler) + { + case error_handler_t::keep: + return s; + + case error_handler_t::strict: + check_utf8(s, context); + return s; + + case error_handler_t::replace: + case error_handler_t::ignore: + default: + if (is_valid_utf8(s)) + { + return s; + } + storage = sanitize_utf8(s, error_handler); + return storage; + } + } + /*! @brief write an integer in the shortest encoding @@ -23488,6 +23831,10 @@ class binary_writer /// the output OutputSinkType oa; + + /// how to treat a string value or object key that is not valid UTF-8 + /// (CBOR, MessagePack, UBJSON, BJData, and BSON; not BON8) + const error_handler_t error_handler = binary_writer_default_error_handler(); }; } // namespace detail @@ -24649,6 +24996,8 @@ NLOHMANN_JSON_NAMESPACE_END // #include +// #include + // #include // #include @@ -24668,14 +25017,6 @@ namespace detail // serialization // /////////////////// -/// how to treat decoding errors -enum class error_handler_t -{ - strict, ///< throw a type_error exception in case of invalid UTF-8 - replace, ///< replace invalid UTF-8 sequences with U+FFFD - ignore ///< ignore invalid UTF-8 sequences -}; - template class serializer { @@ -25466,6 +25807,16 @@ class serializer // EnsureAscii parameter is used, non-ASCII characters if ((codepoint <= 0x1F) || (EnsureAscii && (codepoint >= 0x7F))) { + if (EnsureAscii && error_handler == error_handler_t::keep) + { + // this character was buffered as raw bytes + // below in case it turned out to be part of + // an ill-formed sequence (which is kept as + // is); now that it decoded to a well-formed + // code point, undo that and \u-escape it + // like any other character instead + bytes = bytes_after_last_accept; + } if (codepoint <= 0xFFFF) { write_u_escape(bytes, static_cast(codepoint)); @@ -25564,6 +25915,44 @@ class serializer break; } + case error_handler_t::keep: + { + // the bytes of this (now abandoned) ill-formed + // sequence seen so far are already buffered below + // and are kept unchanged in the output + if (undumped_chars > 0) + { + // the byte that ended the sequence may be OK + // for itself (e.g., a quote that must still be + // escaped, or the lead byte of a well-formed + // code point), so read it again + --i; + } + else + { + // a byte that cannot start a sequence (e.g., + // 0xFF or a stray continuation byte) is kept + // as well + string_buffer[bytes++] = s[i]; + } + + // write buffer and reset index; there must be 13 bytes + // left, as this is the maximal number of bytes to be + // written ("\uxxxx\uxxxx\0") for one code point + if (string_buffer.size() - bytes < 13) + { + put_buffer(string_buffer, bytes); + bytes = 0; + } + + bytes_after_last_accept = bytes; + undumped_chars = 0; + + // continue processing the string + state = UTF8_ACCEPT; + break; + } + default: // LCOV_EXCL_LINE JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE } @@ -25572,9 +25961,12 @@ class serializer default: // decode found yet incomplete multibyte code point { - if (!EnsureAscii) + if (!EnsureAscii || error_handler == error_handler_t::keep) { - // code point will not be escaped - copy byte to buffer + // code point will not be escaped (or will be kept as + // is if it turns out to be ill-formed) - copy byte to + // buffer; dropped again above if it decodes to a + // well-formed code point that needs \u-escaping string_buffer[bytes++] = s[i]; } ++undumped_chars; @@ -25625,6 +26017,14 @@ class serializer break; } + case error_handler_t::keep: + { + // write the ill-formed trailing bytes as is; they were + // buffered above regardless of EnsureAscii + put_buffer(string_buffer, bytes); + break; + } + default: // LCOV_EXCL_LINE JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE } @@ -26732,9 +27132,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // used by the vector-returning to_* overloads template using vector_binary_writer = ::nlohmann::detail::binary_writer>; - template static vector_binary_writer vector_writer(std::vector& v) + template static vector_binary_writer vector_writer( + std::vector& v, const ::nlohmann::detail::error_handler_t error_handler = ::nlohmann::detail::binary_writer_default_error_handler()) { - return vector_binary_writer(::nlohmann::detail::output_vector_sink(v)); + return vector_binary_writer(::nlohmann::detail::output_vector_sink(v), error_handler); } JSON_PRIVATE_UNLESS_TESTED: @@ -32120,11 +32521,12 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec template static basic_json from_binary_impl(InputAdapterType ia, const input_format_t format, const bool strict, const bool allow_exceptions, + const error_handler_t error_handler = error_handler_t::keep, const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error) { basic_json result; detail::json_sax_dom_parser sdp(result, allow_exceptions); - binary_reader reader(std::move(ia), format); + binary_reader reader(std::move(ia), format, error_handler); if (!reader.sax_parse(&sdp, strict, tag_handler)) { result = value_t::discarded; @@ -32142,78 +32544,87 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec public: /// @brief create a CBOR serialization of a given JSON value /// @sa https://json.nlohmann.me/api/basic_json/to_cbor/ - static std::vector to_cbor(const basic_json& j) + static std::vector to_cbor(const basic_json& j, + const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { std::vector result; result.reserve(detail::binary_reserve_hint(j)); - vector_writer(result).write_cbor(j); + vector_writer(result, error_handler).write_cbor(j); return result; } /// @brief create a CBOR serialization of a given JSON value /// @sa https://json.nlohmann.me/api/basic_json/to_cbor/ - static void to_cbor(const basic_json& j, detail::output_adapter o) + static void to_cbor(const basic_json& j, detail::output_adapter o, + const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { - binary_writer(o).write_cbor(j); + binary_writer(o, error_handler).write_cbor(j); } /// @brief create a CBOR serialization of a given JSON value /// @sa https://json.nlohmann.me/api/basic_json/to_cbor/ - static void to_cbor(const basic_json& j, detail::output_adapter o) + static void to_cbor(const basic_json& j, detail::output_adapter o, + const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { - binary_writer(o).write_cbor(j); + binary_writer(o, error_handler).write_cbor(j); } /// @brief create a MessagePack serialization of a given JSON value /// @sa https://json.nlohmann.me/api/basic_json/to_msgpack/ - static std::vector to_msgpack(const basic_json& j) + static std::vector to_msgpack(const basic_json& j, + const error_handler_t error_handler = error_handler_t::keep) { std::vector result; result.reserve(detail::binary_reserve_hint(j)); - vector_writer(result).write_msgpack(j); + vector_writer(result, error_handler).write_msgpack(j); return result; } /// @brief create a MessagePack serialization of a given JSON value /// @sa https://json.nlohmann.me/api/basic_json/to_msgpack/ - static void to_msgpack(const basic_json& j, detail::output_adapter o) + static void to_msgpack(const basic_json& j, detail::output_adapter o, + const error_handler_t error_handler = error_handler_t::keep) { - binary_writer(o).write_msgpack(j); + binary_writer(o, error_handler).write_msgpack(j); } /// @brief create a MessagePack serialization of a given JSON value /// @sa https://json.nlohmann.me/api/basic_json/to_msgpack/ - static void to_msgpack(const basic_json& j, detail::output_adapter o) + static void to_msgpack(const basic_json& j, detail::output_adapter o, + const error_handler_t error_handler = error_handler_t::keep) { - binary_writer(o).write_msgpack(j); + binary_writer(o, error_handler).write_msgpack(j); } /// @brief create a UBJSON serialization of a given JSON value /// @sa https://json.nlohmann.me/api/basic_json/to_ubjson/ static std::vector to_ubjson(const basic_json& j, const bool use_size = false, - const bool use_type = false) + const bool use_type = false, + const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { std::vector result; result.reserve(detail::binary_reserve_hint(j)); - vector_writer(result).write_ubjson(j, use_size, use_type); + vector_writer(result, error_handler).write_ubjson(j, use_size, use_type); return result; } /// @brief create a UBJSON serialization of a given JSON value /// @sa https://json.nlohmann.me/api/basic_json/to_ubjson/ static void to_ubjson(const basic_json& j, detail::output_adapter o, - const bool use_size = false, const bool use_type = false) + const bool use_size = false, const bool use_type = false, + const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { - binary_writer(o).write_ubjson(j, use_size, use_type); + binary_writer(o, error_handler).write_ubjson(j, use_size, use_type); } /// @brief create a UBJSON serialization of a given JSON value /// @sa https://json.nlohmann.me/api/basic_json/to_ubjson/ static void to_ubjson(const basic_json& j, detail::output_adapter o, - const bool use_size = false, const bool use_type = false) + const bool use_size = false, const bool use_type = false, + const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { - binary_writer(o).write_ubjson(j, use_size, use_type); + binary_writer(o, error_handler).write_ubjson(j, use_size, use_type); } /// @brief create a BJData serialization of a given JSON value @@ -32221,11 +32632,12 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec static std::vector to_bjdata(const basic_json& j, const bool use_size = false, const bool use_type = false, - const bjdata_version_t version = bjdata_version_t::draft2) + const bjdata_version_t version = bjdata_version_t::draft2, + const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { std::vector result; result.reserve(detail::binary_reserve_hint(j)); - vector_writer(result).write_ubjson(j, use_size, use_type, true, true, version); + vector_writer(result, error_handler).write_ubjson(j, use_size, use_type, true, true, version); return result; } @@ -32233,42 +32645,47 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/to_bjdata/ static void to_bjdata(const basic_json& j, detail::output_adapter o, const bool use_size = false, const bool use_type = false, - const bjdata_version_t version = bjdata_version_t::draft2) + const bjdata_version_t version = bjdata_version_t::draft2, + const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { - binary_writer(o).write_ubjson(j, use_size, use_type, true, true, version); + binary_writer(o, error_handler).write_ubjson(j, use_size, use_type, true, true, version); } /// @brief create a BJData serialization of a given JSON value /// @sa https://json.nlohmann.me/api/basic_json/to_bjdata/ static void to_bjdata(const basic_json& j, detail::output_adapter o, const bool use_size = false, const bool use_type = false, - const bjdata_version_t version = bjdata_version_t::draft2) + const bjdata_version_t version = bjdata_version_t::draft2, + const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { - binary_writer(o).write_ubjson(j, use_size, use_type, true, true, version); + binary_writer(o, error_handler).write_ubjson(j, use_size, use_type, true, true, version); } /// @brief create a BSON serialization of a given JSON value /// @sa https://json.nlohmann.me/api/basic_json/to_bson/ - static std::vector to_bson(const basic_json& j) + static std::vector to_bson(const basic_json& j, + const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { std::vector result; result.reserve(detail::binary_reserve_hint(j)); - vector_writer(result).write_bson(j); + vector_writer(result, error_handler).write_bson(j); return result; } /// @brief create a BSON serialization of a given JSON value /// @sa https://json.nlohmann.me/api/basic_json/to_bson/ - static void to_bson(const basic_json& j, detail::output_adapter o) + static void to_bson(const basic_json& j, detail::output_adapter o, + const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { - binary_writer(o).write_bson(j); + binary_writer(o, error_handler).write_bson(j); } /// @brief create a BSON serialization of a given JSON value /// @sa https://json.nlohmann.me/api/basic_json/to_bson/ - static void to_bson(const basic_json& j, detail::output_adapter o) + static void to_bson(const basic_json& j, detail::output_adapter o, + const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { - binary_writer(o).write_bson(j); + binary_writer(o, error_handler).write_bson(j); } /// @brief create a BON8 serialization of a given JSON value @@ -32302,9 +32719,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec static basic_json from_cbor(InputType&& i, const bool strict = true, const bool allow_exceptions = true, - const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error) + const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error, + const error_handler_t error_handler = error_handler_t::keep) { - return from_binary_impl(detail::input_adapter(std::forward(i)), input_format_t::cbor, strict, allow_exceptions, tag_handler); + return from_binary_impl(detail::input_adapter(std::forward(i)), input_format_t::cbor, strict, allow_exceptions, error_handler, tag_handler); } /// @brief create a JSON value from an input in CBOR format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -32315,9 +32733,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec static basic_json from_cbor(IteratorType first, SentinelType last, const bool strict = true, const bool allow_exceptions = true, - const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error) + const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error, + const error_handler_t error_handler = error_handler_t::keep) { - return from_binary_impl(detail::input_adapter(std::move(first), std::move(last)), input_format_t::cbor, strict, allow_exceptions, tag_handler); + return from_binary_impl(detail::input_adapter(std::move(first), std::move(last)), input_format_t::cbor, strict, allow_exceptions, error_handler, tag_handler); } template @@ -32338,7 +32757,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool allow_exceptions = true, const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error) { - return from_binary_impl(i.get(), input_format_t::cbor, strict, allow_exceptions, tag_handler); + return from_binary_impl(i.get(), input_format_t::cbor, strict, allow_exceptions, error_handler_t::keep, tag_handler); } /// @brief create a JSON value from an input in MessagePack format @@ -32347,9 +32766,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec JSON_HEDLEY_WARN_UNUSED_RESULT static basic_json from_msgpack(InputType&& i, const bool strict = true, - const bool allow_exceptions = true) + const bool allow_exceptions = true, + const error_handler_t error_handler = error_handler_t::keep) { - return from_binary_impl(detail::input_adapter(std::forward(i)), input_format_t::msgpack, strict, allow_exceptions); + return from_binary_impl(detail::input_adapter(std::forward(i)), input_format_t::msgpack, strict, allow_exceptions, error_handler); } /// @brief create a JSON value from an input in MessagePack format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -32359,9 +32779,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec JSON_HEDLEY_WARN_UNUSED_RESULT static basic_json from_msgpack(IteratorType first, SentinelType last, const bool strict = true, - const bool allow_exceptions = true) + const bool allow_exceptions = true, + const error_handler_t error_handler = error_handler_t::keep) { - return from_binary_impl(detail::input_adapter(std::move(first), std::move(last)), input_format_t::msgpack, strict, allow_exceptions); + return from_binary_impl(detail::input_adapter(std::move(first), std::move(last)), input_format_t::msgpack, strict, allow_exceptions, error_handler); } template @@ -32389,9 +32810,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec JSON_HEDLEY_WARN_UNUSED_RESULT static basic_json from_ubjson(InputType&& i, const bool strict = true, - const bool allow_exceptions = true) + const bool allow_exceptions = true, + const error_handler_t error_handler = error_handler_t::keep) { - return from_binary_impl(detail::input_adapter(std::forward(i)), input_format_t::ubjson, strict, allow_exceptions); + return from_binary_impl(detail::input_adapter(std::forward(i)), input_format_t::ubjson, strict, allow_exceptions, error_handler); } /// @brief create a JSON value from an input in UBJSON format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -32401,9 +32823,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec JSON_HEDLEY_WARN_UNUSED_RESULT static basic_json from_ubjson(IteratorType first, SentinelType last, const bool strict = true, - const bool allow_exceptions = true) + const bool allow_exceptions = true, + const error_handler_t error_handler = error_handler_t::keep) { - return from_binary_impl(detail::input_adapter(std::move(first), std::move(last)), input_format_t::ubjson, strict, allow_exceptions); + return from_binary_impl(detail::input_adapter(std::move(first), std::move(last)), input_format_t::ubjson, strict, allow_exceptions, error_handler); } template @@ -32431,9 +32854,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec JSON_HEDLEY_WARN_UNUSED_RESULT static basic_json from_bjdata(InputType&& i, const bool strict = true, - const bool allow_exceptions = true) + const bool allow_exceptions = true, + const error_handler_t error_handler = error_handler_t::keep) { - return from_binary_impl(detail::input_adapter(std::forward(i)), input_format_t::bjdata, strict, allow_exceptions); + return from_binary_impl(detail::input_adapter(std::forward(i)), input_format_t::bjdata, strict, allow_exceptions, error_handler); } /// @brief create a JSON value from an input in BJData format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -32443,9 +32867,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec JSON_HEDLEY_WARN_UNUSED_RESULT static basic_json from_bjdata(IteratorType first, SentinelType last, const bool strict = true, - const bool allow_exceptions = true) + const bool allow_exceptions = true, + const error_handler_t error_handler = error_handler_t::keep) { - return from_binary_impl(detail::input_adapter(std::move(first), std::move(last)), input_format_t::bjdata, strict, allow_exceptions); + return from_binary_impl(detail::input_adapter(std::move(first), std::move(last)), input_format_t::bjdata, strict, allow_exceptions, error_handler); } /// @brief create a JSON value from an input in BON8 format @@ -32477,9 +32902,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec JSON_HEDLEY_WARN_UNUSED_RESULT static basic_json from_bson(InputType&& i, const bool strict = true, - const bool allow_exceptions = true) + const bool allow_exceptions = true, + const error_handler_t error_handler = error_handler_t::keep) { - return from_binary_impl(detail::input_adapter(std::forward(i)), input_format_t::bson, strict, allow_exceptions); + return from_binary_impl(detail::input_adapter(std::forward(i)), input_format_t::bson, strict, allow_exceptions, error_handler); } /// @brief create a JSON value from an input in BSON format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -32489,9 +32915,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec JSON_HEDLEY_WARN_UNUSED_RESULT static basic_json from_bson(IteratorType first, SentinelType last, const bool strict = true, - const bool allow_exceptions = true) + const bool allow_exceptions = true, + const error_handler_t error_handler = error_handler_t::keep) { - return from_binary_impl(detail::input_adapter(std::move(first), std::move(last)), input_format_t::bson, strict, allow_exceptions); + return from_binary_impl(detail::input_adapter(std::move(first), std::move(last)), input_format_t::bson, strict, allow_exceptions, error_handler); } template @@ -33734,6 +34161,7 @@ struct formatter // NOLINT(cert-dcl58-c #undef JSON_BRACE_INIT_COPY_SEMANTICS #undef JSON_PRECISE_STREAM_POSITION #undef JSON_STRICT_NUL_HANDLING + #undef JSON_STRICT_BINARY_UTF8 #endif // #include diff --git a/tests/src/unit-binary_utf8_error_handler.cpp b/tests/src/unit-binary_utf8_error_handler.cpp new file mode 100644 index 000000000..85505bffb --- /dev/null +++ b/tests/src/unit-binary_utf8_error_handler.cpp @@ -0,0 +1,362 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#include "doctest_compatibility.h" + +#include +using nlohmann::json; + +#include +#include + +namespace +{ + +struct ill_formed_case +{ + const char* name; + std::string bytes; +}; + +// RFC 3629 ill-formed sequences used throughout this file, plus one +// well-formed sequence for contrast +const std::vector ill_formed_cases = +{ + {"overlong", "\xC0\xAE"}, + {"lone_0xFF", "\xFF"}, + {"truncated", "\xE2\x82"}, + {"surrogate", "\xED\xA0\x80"}, +}; + +const std::string valid_sequence = "\xC3\xA9"; // U+00E9, "é" + +using eh = json::error_handler_t; +const std::vector all_handlers = {eh::strict, eh::replace, eh::ignore, eh::keep}; + +// what dump()+parse() produces for a sanitizing error_handler; this is the +// ground truth every binary writer/reader is checked against +std::string dump_and_parse(const std::string& raw, eh error_handler) +{ + return json::parse(json(raw).dump(-1, ' ', false, error_handler)).get(); +} + +} // namespace + +TEST_CASE("UTF-8 error_handler for the binary readers and writers") +{ + SECTION("writers: string value") + { + for (const auto& c : ill_formed_cases) + { + CAPTURE(c.name); + const json jval = c.bytes; + + CHECK_THROWS_AS(json::to_cbor(jval, eh::strict), json::type_error&); + CHECK_THROWS_AS(json::to_msgpack(jval, eh::strict), json::type_error&); + CHECK_THROWS_AS(json::to_ubjson(jval, false, false, eh::strict), json::type_error&); + CHECK_THROWS_AS(json::to_bjdata(jval, false, false, json::bjdata_version_t::draft2, eh::strict), json::type_error&); + { + json jobj; + jobj["k"] = jval; + CHECK_THROWS_AS(json::to_bson(jobj, eh::strict), json::type_error&); + } + + for (const auto h : + { + eh::replace, eh::ignore + }) + { + CAPTURE(static_cast(h)); + const std::string expected = dump_and_parse(c.bytes, h); + + CHECK(json::from_cbor(json::to_cbor(jval, h)).get() == expected); + CHECK(json::from_msgpack(json::to_msgpack(jval, h)).get() == expected); + CHECK(json::from_ubjson(json::to_ubjson(jval, false, false, h)).get() == expected); + CHECK(json::from_bjdata(json::to_bjdata(jval, false, false, json::bjdata_version_t::draft2, h)).get() == expected); + { + json jobj; + jobj["k"] = jval; + const auto bytes = json::to_bson(jobj, h); + CHECK(json::from_bson(bytes)["k"].get() == expected); + } + } + + // keep: the writer passes the ill-formed bytes through unchanged, + // exactly as every binary writer did before this parameter existed + CHECK(json::from_cbor(json::to_cbor(jval, eh::keep)).get() == c.bytes); + CHECK(json::from_msgpack(json::to_msgpack(jval, eh::keep)).get() == c.bytes); + CHECK(json::from_ubjson(json::to_ubjson(jval, false, false, eh::keep)).get() == c.bytes); + CHECK(json::from_bjdata(json::to_bjdata(jval, false, false, json::bjdata_version_t::draft2, eh::keep)).get() == c.bytes); + { + json jobj; + jobj["k"] = jval; + const auto bytes = json::to_bson(jobj, eh::keep); + CHECK(json::from_bson(bytes)["k"].get() == c.bytes); + } + } + } + + SECTION("writers: object key") + { + for (const auto& c : ill_formed_cases) + { + CAPTURE(c.name); + json jobj; + jobj[c.bytes] = 1; + + CHECK_THROWS_AS(json::to_cbor(jobj, eh::strict), json::type_error&); + CHECK_THROWS_AS(json::to_msgpack(jobj, eh::strict), json::type_error&); + CHECK_THROWS_AS(json::to_ubjson(jobj, false, false, eh::strict), json::type_error&); + CHECK_THROWS_AS(json::to_bjdata(jobj, false, false, json::bjdata_version_t::draft2, eh::strict), json::type_error&); + CHECK_THROWS_AS(json::to_bson(jobj, eh::strict), json::type_error&); + + for (const auto h : + { + eh::replace, eh::ignore + }) + { + CAPTURE(static_cast(h)); + const std::string expected = dump_and_parse(c.bytes, h); + + CHECK(json::from_cbor(json::to_cbor(jobj, h)).begin().key() == expected); + CHECK(json::from_msgpack(json::to_msgpack(jobj, h)).begin().key() == expected); + CHECK(json::from_ubjson(json::to_ubjson(jobj, false, false, h)).begin().key() == expected); + CHECK(json::from_bjdata(json::to_bjdata(jobj, false, false, json::bjdata_version_t::draft2, h)).begin().key() == expected); + CHECK(json::from_bson(json::to_bson(jobj, h)).begin().key() == expected); + } + + // keep: object keys round-trip unchanged too + CHECK(json::from_cbor(json::to_cbor(jobj, eh::keep)).begin().key() == c.bytes); + CHECK(json::from_msgpack(json::to_msgpack(jobj, eh::keep)).begin().key() == c.bytes); + CHECK(json::from_ubjson(json::to_ubjson(jobj, false, false, eh::keep)).begin().key() == c.bytes); + CHECK(json::from_bjdata(json::to_bjdata(jobj, false, false, json::bjdata_version_t::draft2, eh::keep)).begin().key() == c.bytes); + CHECK(json::from_bson(json::to_bson(jobj, eh::keep)).begin().key() == c.bytes); + } + } + + SECTION("readers: string value") + { + for (const auto& c : ill_formed_cases) + { + CAPTURE(c.name); + + // bytes produced the lenient (keep) way, as any binary reader + // accepted them before this parameter existed + const auto cbor_bytes = json::to_cbor(json(c.bytes), eh::keep); + const auto msgpack_bytes = json::to_msgpack(json(c.bytes)); // to_msgpack has no error_handler; always pass-through + const auto ubjson_bytes = json::to_ubjson(json(c.bytes), false, false, eh::keep); + const auto bjdata_bytes = json::to_bjdata(json(c.bytes), false, false, json::bjdata_version_t::draft2, eh::keep); + const auto bson_bytes = [&c] + { + json jobj; + jobj["k"] = c.bytes; + return json::to_bson(jobj, eh::keep); + }(); + + // keep (the default): bytes are kept unchanged + CHECK(json::from_cbor(cbor_bytes).get() == c.bytes); + CHECK(json::from_msgpack(msgpack_bytes).get() == c.bytes); + CHECK(json::from_ubjson(ubjson_bytes).get() == c.bytes); + CHECK(json::from_bjdata(bjdata_bytes).get() == c.bytes); + CHECK(json::from_bson(bson_bytes)["k"].get() == c.bytes); + + // strict: parse_error.113, discarded (not thrown) when allow_exceptions is false + CHECK_THROWS_AS(json::from_cbor(cbor_bytes, true, true, json::cbor_tag_handler_t::error, eh::strict), json::parse_error&); + CHECK(json::from_cbor(cbor_bytes, true, false, json::cbor_tag_handler_t::error, eh::strict).is_discarded()); + CHECK_THROWS_AS(json::from_msgpack(msgpack_bytes, true, true, eh::strict), json::parse_error&); + CHECK(json::from_msgpack(msgpack_bytes, true, false, eh::strict).is_discarded()); + CHECK_THROWS_AS(json::from_ubjson(ubjson_bytes, true, true, eh::strict), json::parse_error&); + CHECK(json::from_ubjson(ubjson_bytes, true, false, eh::strict).is_discarded()); + CHECK_THROWS_AS(json::from_bjdata(bjdata_bytes, true, true, eh::strict), json::parse_error&); + CHECK(json::from_bjdata(bjdata_bytes, true, false, eh::strict).is_discarded()); + CHECK_THROWS_AS(json::from_bson(bson_bytes, true, true, eh::strict), json::parse_error&); + CHECK(json::from_bson(bson_bytes, true, false, eh::strict).is_discarded()); + + // replace / ignore: match what dump() would have sanitized the same bytes to + for (const auto h : + { + eh::replace, eh::ignore + }) + { + CAPTURE(static_cast(h)); + const std::string expected = dump_and_parse(c.bytes, h); + + CHECK(json::from_cbor(cbor_bytes, true, true, json::cbor_tag_handler_t::error, h).get() == expected); + CHECK(json::from_msgpack(msgpack_bytes, true, true, h).get() == expected); + CHECK(json::from_ubjson(ubjson_bytes, true, true, h).get() == expected); + CHECK(json::from_bjdata(bjdata_bytes, true, true, h).get() == expected); + CHECK(json::from_bson(bson_bytes, true, true, h)["k"].get() == expected); + } + } + } + + SECTION("readers: object key") + { + for (const auto& c : ill_formed_cases) + { + CAPTURE(c.name); + + json jobj; + jobj[c.bytes] = 1; + const auto cbor_bytes = json::to_cbor(jobj, eh::keep); + const auto msgpack_bytes = json::to_msgpack(jobj); + const auto ubjson_bytes = json::to_ubjson(jobj, false, false, eh::keep); + const auto bjdata_bytes = json::to_bjdata(jobj, false, false, json::bjdata_version_t::draft2, eh::keep); + const auto bson_bytes = json::to_bson(jobj, eh::keep); + + CHECK(json::from_cbor(cbor_bytes).begin().key() == c.bytes); + CHECK(json::from_msgpack(msgpack_bytes).begin().key() == c.bytes); + CHECK(json::from_ubjson(ubjson_bytes).begin().key() == c.bytes); + CHECK(json::from_bjdata(bjdata_bytes).begin().key() == c.bytes); + CHECK(json::from_bson(bson_bytes).begin().key() == c.bytes); + + CHECK_THROWS_AS(json::from_cbor(cbor_bytes, true, true, json::cbor_tag_handler_t::error, eh::strict), json::parse_error&); + CHECK_THROWS_AS(json::from_msgpack(msgpack_bytes, true, true, eh::strict), json::parse_error&); + CHECK_THROWS_AS(json::from_ubjson(ubjson_bytes, true, true, eh::strict), json::parse_error&); + CHECK_THROWS_AS(json::from_bjdata(bjdata_bytes, true, true, eh::strict), json::parse_error&); + CHECK_THROWS_AS(json::from_bson(bson_bytes, true, true, eh::strict), json::parse_error&); + + for (const auto h : + { + eh::replace, eh::ignore + }) + { + CAPTURE(static_cast(h)); + const std::string expected = dump_and_parse(c.bytes, h); + + CHECK(json::from_cbor(cbor_bytes, true, true, json::cbor_tag_handler_t::error, h).begin().key() == expected); + CHECK(json::from_msgpack(msgpack_bytes, true, true, h).begin().key() == expected); + CHECK(json::from_ubjson(ubjson_bytes, true, true, h).begin().key() == expected); + CHECK(json::from_bjdata(bjdata_bytes, true, true, h).begin().key() == expected); + CHECK(json::from_bson(bson_bytes, true, true, h).begin().key() == expected); + } + } + } + + SECTION("well-formed UTF-8 is unaffected by error_handler") + { + const json jval = valid_sequence; + json jobj; + jobj[valid_sequence] = valid_sequence; + + for (const auto h : all_handlers) + { + CAPTURE(static_cast(h)); + + CHECK(json::from_cbor(json::to_cbor(jval, h)).get() == valid_sequence); + CHECK(json::from_msgpack(json::to_msgpack(jval, h)).get() == valid_sequence); + CHECK(json::from_ubjson(json::to_ubjson(jval, false, false, h)).get() == valid_sequence); + CHECK(json::from_bjdata(json::to_bjdata(jval, false, false, json::bjdata_version_t::draft2, h)).get() == valid_sequence); + CHECK(json::from_bson(json::to_bson(jobj, h)).begin().key() == valid_sequence); + + CHECK(json::from_cbor(json::to_cbor(jval, eh::keep), true, true, json::cbor_tag_handler_t::error, h).get() == valid_sequence); + CHECK(json::from_msgpack(json::to_msgpack(jval), true, true, h).get() == valid_sequence); + } + } + + SECTION("dump() with error_handler_t::keep writes raw bytes as is") + { + for (const auto& c : ill_formed_cases) + { + CAPTURE(c.name); + + const json jval = c.bytes; + const std::string dumped = jval.dump(-1, ' ', false, eh::keep); + CHECK(dumped.find(c.bytes) != std::string::npos); + + // even with ensure_ascii, the ill-formed bytes are written as is + const std::string dumped_ascii = jval.dump(-1, ' ', true, eh::keep); + CHECK(dumped_ascii.find(c.bytes) != std::string::npos); + } + + // well-formed characters around an ill-formed sequence are still + // escaped as usual under ensure_ascii + const json mixed = valid_sequence + ill_formed_cases[1].bytes; // "é" + lone 0xFF + const std::string dumped_mixed = mixed.dump(-1, ' ', true, eh::keep); + CHECK(dumped_mixed.find("\\u00e9") != std::string::npos); + CHECK(dumped_mixed.find(ill_formed_cases[1].bytes) != std::string::npos); + + // the byte that ends an ill-formed sequence is read again, so a quote, + // a backslash, or a control character after it is still escaped, and + // a well-formed code point after it is escaped under ensure_ascii + for (const bool ensure_ascii : + { + false, true + }) + { + CAPTURE(ensure_ascii); + CHECK(json("\xC3\"").dump(-1, ' ', ensure_ascii, eh::keep) == "\"\xC3\\\"\""); + CHECK(json("\xC3\\").dump(-1, ' ', ensure_ascii, eh::keep) == "\"\xC3\\\\\""); + CHECK(json("\xC3\n").dump(-1, ' ', ensure_ascii, eh::keep) == "\"\xC3\\n\""); + CHECK(json("\xE2\x82\"").dump(-1, ' ', ensure_ascii, eh::keep) == "\"\xE2\x82\\\"\""); + CHECK(json("\xFF\"").dump(-1, ' ', ensure_ascii, eh::keep) == "\"\xFF\\\"\""); + CHECK(json("a\xE2\x82").dump(-1, ' ', ensure_ascii, eh::keep) == "\"a\xE2\x82\""); + } + CHECK(json("\xC3\xC3\xA9").dump(-1, ' ', false, eh::keep) == "\"\xC3\xC3\xA9\""); + CHECK(json("\xC3\xC3\xA9").dump(-1, ' ', true, eh::keep) == "\"\xC3\\u00e9\""); + } + + SECTION("to_msgpack defaults to keep; to_bon8 is not affected by error_handler") + { + const json jval = ill_formed_cases[1].bytes; // lone 0xFF + + // to_msgpack's error_handler defaults to keep, as MessagePack's spec + // allows any bytes in a str, so the bytes are passed through + CHECK(json::to_msgpack(jval) == json::to_msgpack(jval, eh::keep)); + CHECK(json::from_msgpack(json::to_msgpack(jval)).get() == ill_formed_cases[1].bytes); + + // the diagnostics context of an ill-formed key is the object + json jobj; + jobj["\xFF"] = 1; + CHECK_THROWS_WITH_AS(json::to_msgpack(jobj, eh::strict), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xFF", json::type_error&); + + // to_bon8 has no error_handler parameter; UTF-8 is structural for + // BON8, so it always rejects ill-formed input + CHECK_THROWS_AS(json::to_bon8(jval), json::type_error&); + } + + SECTION("allow_exceptions=false with error_handler_t::strict discards the value") + { + const auto bytes = json::to_cbor(json(ill_formed_cases[0].bytes), eh::keep); + const json result = json::from_cbor(bytes, true, false, json::cbor_tag_handler_t::error, eh::strict); + CHECK(result.is_discarded()); + } + + SECTION("default parameters are unchanged") + { + const json jval = ill_formed_cases[0].bytes; + + // to_*: the default error_handler is keep, so ill-formed bytes are + // written unchanged, exactly as in release 3.12.0 (it is strict only + // if JSON_STRICT_BINARY_UTF8 is enabled, see + // unit-binary_utf8_strict.cpp) + CHECK(json::to_cbor(jval) == json::to_cbor(jval, eh::keep)); + CHECK(json::to_ubjson(jval) == json::to_ubjson(jval, false, false, eh::keep)); + CHECK(json::to_bjdata(jval) == json::to_bjdata(jval, false, false, json::bjdata_version_t::draft2, eh::keep)); + { + json jobj; + jobj["k"] = jval; + CHECK(json::to_bson(jobj) == json::to_bson(jobj, eh::keep)); + } + + // from_*: the default error_handler is keep, so ill-formed bytes are + // still accepted unchanged, exactly as in release 3.12.0 + const auto cbor_bytes = json::to_cbor(jval, eh::keep); + CHECK(json::from_cbor(cbor_bytes).get() == ill_formed_cases[0].bytes); + const auto ubjson_bytes = json::to_ubjson(jval, false, false, eh::keep); + CHECK(json::from_ubjson(ubjson_bytes).get() == ill_formed_cases[0].bytes); + const auto bjdata_bytes = json::to_bjdata(jval, false, false, json::bjdata_version_t::draft2, eh::keep); + CHECK(json::from_bjdata(bjdata_bytes).get() == ill_formed_cases[0].bytes); + const auto msgpack_bytes = json::to_msgpack(jval); + CHECK(json::from_msgpack(msgpack_bytes).get() == ill_formed_cases[0].bytes); + json bson_obj; + bson_obj["k"] = jval; + const auto bson_bytes = json::to_bson(bson_obj, eh::keep); + CHECK(json::from_bson(bson_bytes)["k"].get() == ill_formed_cases[0].bytes); + } +} diff --git a/tests/src/unit-binary_utf8_strict.cpp b/tests/src/unit-binary_utf8_strict.cpp index c83b5938c..2ac8aabf4 100644 --- a/tests/src/unit-binary_utf8_strict.cpp +++ b/tests/src/unit-binary_utf8_strict.cpp @@ -100,11 +100,23 @@ TEST_CASE("JSON_STRICT_BINARY_UTF8 (see #5529, #5651)") CHECK_THROWS_WITH_AS(json::to_bson(json{{"\xFF", 1}}), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xFF", json::type_error&); } + SECTION("an explicit error_handler overrides the default") + { + // the macro only changes the default of the error_handler parameter + CHECK(json::to_cbor(json("\xFF"), json::error_handler_t::keep) == std::vector({0x61, 0xff})); + CHECK(json::to_ubjson(json("\xFF"), false, false, json::error_handler_t::keep) == std::vector({'S', 'i', 1, 0xff})); + CHECK(json::to_bjdata(json("\xFF"), false, false, json::bjdata_version_t::draft2, json::error_handler_t::keep) == std::vector({'S', 'i', 1, 0xff})); + CHECK(json::from_bson(json::to_bson(json{{"s", "\xFF"}}, json::error_handler_t::keep)) == json{{"s", "\xFF"}}); + CHECK(json::to_cbor(json("\xFF"), json::error_handler_t::replace) == std::vector({0x63, 0xef, 0xbf, 0xbd})); + } + SECTION("MessagePack and BON8 are unaffected") { - // MessagePack allows any bytes in a str, so to_msgpack() writes them as - // is; BON8 always checks, because the lead bytes mark where strings end + // MessagePack allows any bytes in a str, so to_msgpack() still + // defaults to keep (strict only if passed explicitly); BON8 always + // checks, because the lead bytes mark where strings end CHECK(json::to_msgpack(json("\xFF")) == std::vector({0xa1, 0xff})); + CHECK_THROWS_AS(json::to_msgpack(json("\xFF"), json::error_handler_t::strict), json::type_error&); CHECK_THROWS_AS(json::to_bon8(json("\xFF")), json::type_error&); } } From 0c4462676dfe0787ed6788079821323a40416234 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 4 Oct 2026 12:23:53 +0200 Subject: [PATCH 21/22] Document options to reduce compile times (#5611) * Document options to reduce compile times Add an integration page that collects the ways to reduce compile times with measurements: json_fwd.hpp in headers, JSON_NO_AUTOMATIC_UDLS, explicit instantiation with extern template, modules, and precompiled headers, and notes that JSON_NO_IO and JSON_USE_GLOBAL_UDLS have no measurable effect. Signed-off-by: Niels Lohmann * Include only json_literals.hpp in the JSON_NO_AUTOMATIC_UDLS examples json_literals.hpp includes json.hpp itself, so including both is redundant. Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann --- .../docs/api/macros/json_no_automatic_udls.md | 3 +- docs/mkdocs/docs/integration/compile_times.md | 149 ++++++++++++++++++ docs/mkdocs/docs/integration/index.md | 2 +- docs/mkdocs/mkdocs.yml | 1 + 4 files changed, 153 insertions(+), 2 deletions(-) create mode 100644 docs/mkdocs/docs/integration/compile_times.md diff --git a/docs/mkdocs/docs/api/macros/json_no_automatic_udls.md b/docs/mkdocs/docs/api/macros/json_no_automatic_udls.md index acb6a356e..1235d634d 100644 --- a/docs/mkdocs/docs/api/macros/json_no_automatic_udls.md +++ b/docs/mkdocs/docs/api/macros/json_no_automatic_udls.md @@ -43,9 +43,9 @@ By default, `#!cpp JSON_NO_AUTOMATIC_UDLS` is not defined, and ` // this file uses the literals, so it includes them explicitly + // (the header includes itself) #include int main() @@ -62,6 +62,7 @@ By default, `#!cpp JSON_NO_AUTOMATIC_UDLS` is not defined, and `` pays for parsing the header and instantiating what it uses. This page lists the options to reduce +that cost, ordered by how much they typically save. + +!!! info "Measurements" + + The numbers below are medians of nine runs compiling a single translation unit with `-std=c++17 -c` against the + single-header version, with Apple clang and GCC 16 on macOS (Apple silicon). They show the order of magnitude to + expect; measure your own code before and after a change. + +## Include `json_fwd.hpp` in headers + +Header files that only need to *name* the `json` type — for function declarations, members held by pointer or +reference, or friend declarations — can include `` instead of ``. It only +forward-declares `basic_json`, `json`, `ordered_json`, `json_pointer`, and `adl_serializer`. The translation units that +actually use the values then include ``. + +```cpp title="person.hpp" +#pragma once +#include + +struct person; +void to_json(nlohmann::json& j, const person& p); +void from_json(const nlohmann::json& j, person& p); +``` + +```cpp title="person.cpp" +#include "person.hpp" +#include + +void to_json(nlohmann::json& j, const person& p) { /* ... */ } +void from_json(const nlohmann::json& j, person& p) { /* ... */ } +``` + +| Compiler | `json.hpp` (`-O0`) | `json_fwd.hpp` (`-O0`) | Change | +|-------------|-------------------:|-----------------------:|-------:| +| Apple clang | 704 ms | 329 ms | −53% | +| GCC 16 | 779 ms | 242 ms | −69% | + +This is the most effective option, because it avoids the full header in every translation unit that includes +*your* headers. + +## Opt out of the automatic user-defined string literals + +The user-defined string literals [`operator""_json`](../api/operator_literal_json.md) and +[`operator""_json_pointer`](../api/operator_literal_json_pointer.md) are ordinary inline functions whose bodies call the +parser. As `` includes them by default, every translation unit instantiates the parser, even if it +never parses anything itself. + +Define [`JSON_NO_AUTOMATIC_UDLS`](../api/macros/json_no_automatic_udls.md) for the whole project and include +`` instead of `` in the files that use the literals (it includes +`` itself): + +```cmake +target_compile_definitions(my_target PRIVATE JSON_NO_AUTOMATIC_UDLS) +``` + +```cpp +#include // only where "..."_json is used; includes +``` + +The saving applies to translation units that do not parse JSON, for example ones that define types and their +conversions or only pass `json` values around: + +| Compiler | Translation unit | Default (`-O0` / `-O2`) | `JSON_NO_AUTOMATIC_UDLS` (`-O0` / `-O2`) | Change | +|-------------|------------------|------------------------:|-----------------------------------------:|------------:| +| Apple clang | model | 776 ms / 846 ms | 629 ms / 692 ms | −19% / −18% | +| GCC 16 | model | 1022 ms / 1120 ms | 882 ms / 965 ms | −14% / −14% | +| Apple clang | parsing | 992 ms / 1815 ms | 1006 ms / 1823 ms | +1% / 0% | +| GCC 16 | parsing | 2018 ms / 3420 ms | 1990 ms / 3454 ms | −1% / +1% | + +Translation units that include only the header save up to a third. Translation units that parse anyway instantiate +the parser regardless and see no difference. + +## Instantiate `basic_json` once + +Each translation unit instantiates the member functions of `nlohmann::json` it uses. An explicit instantiation +declaration tells the compiler that the non-template members are instantiated elsewhere, so it can skip them: + +```cpp title="json_instance.hpp" +#pragma once +#include + +extern template class nlohmann::basic_json<>; +``` + +```cpp title="json_instance.cpp" +#include "json_instance.hpp" + +template class nlohmann::basic_json<>; +``` + +Include `json_instance.hpp` instead of `` and compile and link `json_instance.cpp` once. + +| Compiler | Translation unit | Default (`-O0` / `-O2`) | `extern template` (`-O0` / `-O2`) | Change | +|-------------|---------------------|------------------------:|----------------------------------:|------------:| +| Apple clang | parsing | 992 ms / 1815 ms | 953 ms / 1625 ms | −4% / −10% | +| GCC 16 | parsing | 2018 ms / 3420 ms | 1522 ms / 2728 ms | −25% / −20% | +| Apple clang | `json_instance.cpp` | — | 2166 ms / 4660 ms | — | +| GCC 16 | `json_instance.cpp` | — | 5085 ms / 10616 ms | — | + +Notes: + +- The saving grows with the number of translation units that use `json`, while the instantiation translation unit is + compiled only once (and is rarely recompiled, as it does not depend on your code). +- Member function templates (such as `get()`, `parse(InputType&&)`, or `value(key, default)`) are not covered by + the explicit instantiation and are still instantiated where they are used. +- The declaration covers exactly `nlohmann::json`. Add the same lines for `nlohmann::ordered_json` + (`nlohmann::basic_json`) or your own `basic_json` specializations if you use them. + +## Use C++20 modules + +With a toolchain that supports named modules, `import nlohmann.json;` compiles the library once into a module and +avoids parsing the header in every translation unit. See [Modules](../features/modules.md) for requirements and known +issues. Module support is experimental and currently depends heavily on the compiler version. + +## Use precompiled headers + +Build systems can precompile `` together with other stable headers, for example with CMake's +[`target_precompile_headers`](https://cmake.org/cmake/help/latest/command/target_precompile_headers.html): + +```cmake +target_precompile_headers(my_target PRIVATE ) +``` + +This removes the cost of parsing the header, but not of instantiating templates in each translation unit, so it +combines well with the options above. + +## Options without effect on compile times + +Some configuration macros change what the library declares, but do not measurably change compile times: + +| Macro | Apple clang, model (`-O0` / `-O2`) | GCC 16, model (`-O0` / `-O2`) | +|------------------------------------------------------------------------|-----------------------------------:|------------------------------:| +| default | 776 ms / 846 ms | 1022 ms / 1120 ms | +| [`JSON_NO_IO`](../api/macros/json_no_io.md) | 764 ms / 836 ms | 1022 ms / 1117 ms | +| [`JSON_USE_GLOBAL_UDLS`](../api/macros/json_use_global_udls.md)`=0` | 763 ms / 852 ms | 1019 ms / 1106 ms | + +`JSON_USE_GLOBAL_UDLS` only controls *where* the literals are declared; to avoid their cost, use +`JSON_NO_AUTOMATIC_UDLS` instead. + +## See also + +- [`JSON_NO_AUTOMATIC_UDLS`](../api/macros/json_no_automatic_udls.md) - do not include the user-defined string + literals automatically +- [Modules](../features/modules.md) - C++20 module support +- [Header only](index.md) - including the library diff --git a/docs/mkdocs/docs/integration/index.md b/docs/mkdocs/docs/integration/index.md index 5c3995a95..846c15fdc 100644 --- a/docs/mkdocs/docs/integration/index.md +++ b/docs/mkdocs/docs/integration/index.md @@ -45,7 +45,7 @@ Clang). You can further use file [`single_include/nlohmann/json_fwd.hpp`](https://github.com/nlohmann/json/blob/develop/single_include/nlohmann/json_fwd.hpp) -for forward declarations, and file +for forward declarations (see [Compile times](compile_times.md)), and file [`single_include/nlohmann/json_literals.hpp`](https://github.com/nlohmann/json/blob/develop/single_include/nlohmann/json_literals.hpp) for the user-defined string literals if you define [`JSON_NO_AUTOMATIC_UDLS`](../api/macros/json_no_automatic_udls.md). diff --git a/docs/mkdocs/mkdocs.yml b/docs/mkdocs/mkdocs.yml index 3cb25ba9b..dcb94864d 100644 --- a/docs/mkdocs/mkdocs.yml +++ b/docs/mkdocs/mkdocs.yml @@ -108,6 +108,7 @@ nav: - integration/cmake.md - integration/package_managers.md - integration/pkg-config.md + - integration/compile_times.md - API Documentation: - basic_json: - 'Overview': api/basic_json/index.md From d8d47be4a5a46da22b3d8deee8e4e0b114aaf0e7 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 4 Oct 2026 17:18:19 +0200 Subject: [PATCH 22/22] Fix CI on develop after merging the ready-to-merge PRs (#5754) * Keep the serializer conversion for objects whose keys cannot be converted #5591 added a test converting nlohmann::json into a basic_json whose string type cannot be constructed from std::string. That instantiates convert_iteratively(), whose members.emplace_back(next.key(), ...) needs exactly that key conversion, and broke the build of unit-alt-string. Dispatch on the key's constructibility and leave such conversions to the serializers, as the levels above the nesting bound already do (#3425). Signed-off-by: Niels Lohmann * Fix the remaining CI failures on develop - unit-wstring: with a 16-bit wchar_t (Windows), a lone surrogate is reported as the ill-formed byte 0xFF since #5704; the std::wstring expectations still had the previous . - ci_single_binaries: json_literals.hpp (#5610) and json.hpp include each other on purpose, and IWYU, not following the cycle, asks to replace json.hpp with json_fwd.hpp. Report its findings without failing the build, as already done for json.hpp. Signed-off-by: Niels Lohmann * Fix the library warnings and noexcept specifications from the merged PRs - binary_reader: rename the error_handler constructor parameter, which shadowed the member (-Wshadow, -Wshadow-field-in-constructor; #5746) - basic_json(copy_construct_tag, ...): declare it noexcept when copying the base class is (GCC 16 -Wnoexcept; #5690) - the scalar-on-left legacy comparison operators: noexcept only when converting the scalar is, like their member counterparts (#5682, #5751) - compare_leaves: use std::is_eq/is_lt/is_gt instead of comparing a std::partial_ordering with 0 (-Wzero-as-null-pointer-constant; #5686) - serializer: silence MSVC C4127 for the EnsureAscii template parameter (#5741, #5746) - clang-tidy: return the sanitized reference in binary_writer, take the key of ordered_map::find_impl by const reference (#5727), and mark the switches over parse_array_index (#5728) - ordered_map: keep for std::allocator (IWYU) Signed-off-by: Niels Lohmann * Split unit-conversions.cpp so MinGW can link it clang 18 with the MinGW linker failed to link test-conversions_cpp17 ("relocation truncated to fit: IMAGE_REL_AMD64_REL32"). As windows.yml recommends, keep the objects small by splitting the test file. Signed-off-by: Niels Lohmann * Fix the tests added by the merged PRs for all CI configurations - discard the results of dump() and from_*() in CHECK_THROWS with utils::ignore_return_value (GCC -Werror=unused-result) - give unit-bson's huge_string_t a default constructor (MSVC C2512, GCC 5, clang 3.5) - unit-disabled_exceptions: use the literals namespace when the global UDLs are off (ci_test_noglobaludls; #5700) - unit-binary_utf8_strict: expect the JSON pointer prefix with JSON_DIAGNOSTICS (#5741) - skip the tests that rely on exceptions under JSON_NOEXCEPTION (#5678, #5732) - clang-tidy and clang -Werror: static test data, CAPTURE(...);, const-correctness, use-after-move alias, unused conversion operator, a missing include Signed-off-by: Niels Lohmann * Title the macro examples and add JSON_STRICT_BINARY_UTF8 to the docset The documentation style check requires "Example: ..." titles on pages with several examples (#5741, #5591) and a docset entry for every macro page. Signed-off-by: Niels Lohmann * Regenerate BUILD.bazel and nlohmann_json.natvis Signed-off-by: Niels Lohmann #5746 added detail/output/error_handler.hpp and #5741 the json_abi_sbu8 ABI tag. * Install libidn11 for the CMake 3.5.0 binary in ci_cmake_flags Signed-off-by: Niels Lohmann #5733 moved ci_cmake_options from ubuntu:focal to ubuntu:24.04, which no longer ships libidn.so.11; the CMake 3.5.0 release binary links against it, so every ci_cmake_flags run has failed since. Install focal's libidn11 package for that matrix entry only. * Suppress Infer's false STACK_VARIABLE_ADDRESS_ESCAPE in get_impl get_impl() returns its local by value. A test added by the merged PRs instantiates it with a type Infer misreads, so ci_infer reported the 2021 code for the first time. Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann --- .github/workflows/ubuntu.yml | 7 + BUILD.bazel | 1 + cmake/ci.cmake | 10 +- docs/docset/docSet.sql | 1 + .../api/macros/json_strict_binary_utf8.md | 4 +- .../macros/json_use_implicit_conversions.md | 4 +- .../nlohmann/detail/input/binary_reader.hpp | 6 +- include/nlohmann/detail/json_pointer.hpp | 4 + .../nlohmann/detail/output/binary_writer.hpp | 6 +- include/nlohmann/detail/output/serializer.hpp | 8 + include/nlohmann/json.hpp | 56 +- include/nlohmann/ordered_map.hpp | 4 +- nlohmann_json.natvis | 3840 +++++++++++++++++ single_include/nlohmann/json.hpp | 84 +- tests/src/test_utils.hpp | 3 +- tests/src/unit-allocator.cpp | 2 +- tests/src/unit-binary_utf8_error_handler.cpp | 120 +- tests/src/unit-binary_utf8_strict.cpp | 36 +- tests/src/unit-bjdata.cpp | 2 +- tests/src/unit-bon8.cpp | 2 + tests/src/unit-bson.cpp | 4 +- tests/src/unit-cbor.cpp | 8 +- tests/src/unit-class_parser.cpp | 6 +- tests/src/unit-comparison.cpp | 4 +- ...-conversions.cpp => unit-conversions1.cpp} | 805 ---- tests/src/unit-conversions2.cpp | 868 ++++ tests/src/unit-custom-base-class.cpp | 2 +- tests/src/unit-disabled_exceptions.cpp | 3 + tests/src/unit-element_access2.cpp | 9 +- tests/src/unit-modifiers.cpp | 2 + tests/src/unit-msgpack.cpp | 2 +- tests/src/unit-ordered_map.cpp | 6 +- tests/src/unit-ubjson.cpp | 2 +- tests/src/unit-wstring.cpp | 4 +- 34 files changed, 4978 insertions(+), 947 deletions(-) rename tests/src/{unit-conversions.cpp => unit-conversions1.cpp} (52%) create mode 100644 tests/src/unit-conversions2.cpp diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index f5addb92e..1e4a72870 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -111,6 +111,13 @@ jobs: steps: - name: Install build-essential run: apt-get update ; apt-get install -y build-essential unzip wget git + # the CMake 3.5.0 release binary that ci_cmake_flags downloads links + # against libidn.so.11, which Ubuntu 24.04 no longer ships + - name: Install libidn11 for CMake 3.5.0 + if: matrix.target == 'ci_cmake_flags' + run: | + wget -q http://archive.ubuntu.com/ubuntu/pool/main/libi/libidn/libidn11_1.33-2.2ubuntu2_amd64.deb + dpkg -i libidn11_1.33-2.2ubuntu2_amd64.deb - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false diff --git a/BUILD.bazel b/BUILD.bazel index a9b09fd79..3dfffad1e 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -57,6 +57,7 @@ cc_library( "include/nlohmann/detail/meta/type_traits.hpp", "include/nlohmann/detail/meta/void_t.hpp", "include/nlohmann/detail/output/binary_writer.hpp", + "include/nlohmann/detail/output/error_handler.hpp", "include/nlohmann/detail/output/output_adapters.hpp", "include/nlohmann/detail/output/serializer.hpp", "include/nlohmann/detail/recursion_depth_limit.hpp", diff --git a/cmake/ci.cmake b/cmake/ci.cmake index 2aceb5afd..d0af68786 100644 --- a/cmake/ci.cmake +++ b/cmake/ci.cmake @@ -596,8 +596,9 @@ foreach(SRC_FILE ${SRC_FILES}) add_executable(single_${RELATIVE_SRC_FILE} EXCLUDE_FROM_ALL ${PROJECT_BINARY_DIR}/src_single/${RELATIVE_SRC_FILE}.cpp) target_include_directories(single_${RELATIVE_SRC_FILE} PRIVATE ${PROJECT_SOURCE_DIR}/include) target_compile_features(single_${RELATIVE_SRC_FILE} PRIVATE cxx_std_11) - if(RELATIVE_SRC_FILE STREQUAL "json") - # see below: report json.hpp's diagnostics without --error, so they do not fail the build + if(RELATIVE_SRC_FILE STREQUAL "json" OR RELATIVE_SRC_FILE STREQUAL "json_literals") + # see below: report the diagnostics of json.hpp and json_literals.hpp without --error, so they + # do not fail the build set_property(TARGET single_${RELATIVE_SRC_FILE} PROPERTY CXX_INCLUDE_WHAT_YOU_USE ${IWYU_TOOL} -Xiwyu --max_line_length=300) else() set_property(TARGET single_${RELATIVE_SRC_FILE} PROPERTY CXX_INCLUDE_WHAT_YOU_USE "${iwyu_path_and_options}") @@ -611,7 +612,10 @@ foreach(SRC_FILE ${SRC_FILES}) # reporting its diagnostics (informational, via CXX_INCLUDE_WHAT_YOU_USE above) but exclude it # from the hard gate below so a fresh IWYU/compiler combination does not fail this target on a # nondeterministic suggestion for a header that already re-exports everything on purpose. - if(NOT RELATIVE_SRC_FILE STREQUAL "json") + # json_literals.hpp and json.hpp include each other on purpose (json.hpp includes it at its end + # unless JSON_NO_AUTOMATIC_UDLS is defined), and IWYU, not following the cycle, suggests replacing + # json.hpp with json_fwd.hpp although the literals need the complete basic_json; exclude it, too. + if(NOT RELATIVE_SRC_FILE STREQUAL "json" AND NOT RELATIVE_SRC_FILE STREQUAL "json_literals") list(APPEND single_binaries_tus src_single/${RELATIVE_SRC_FILE}.cpp) endif() endforeach() diff --git a/docs/docset/docSet.sql b/docs/docset/docSet.sql index 09802e43e..801ade93a 100644 --- a/docs/docset/docSet.sql +++ b/docs/docset/docSet.sql @@ -242,6 +242,7 @@ INSERT INTO searchIndex(name, type, path) VALUES ('JSON_NO_THREAD_LOCAL', 'Macro INSERT INTO searchIndex(name, type, path) VALUES ('JSON_PRECISE_STREAM_POSITION', 'Macro', 'api/macros/json_precise_stream_position/index.html'); INSERT INTO searchIndex(name, type, path) VALUES ('JSON_SKIP_LIBRARY_VERSION_CHECK', 'Macro', 'api/macros/json_skip_library_version_check/index.html'); INSERT INTO searchIndex(name, type, path) VALUES ('JSON_SKIP_UNSUPPORTED_COMPILER_CHECK', 'Macro', 'api/macros/json_skip_unsupported_compiler_check/index.html'); +INSERT INTO searchIndex(name, type, path) VALUES ('JSON_STRICT_BINARY_UTF8', 'Macro', 'api/macros/json_strict_binary_utf8/index.html'); INSERT INTO searchIndex(name, type, path) VALUES ('JSON_STRICT_NUL_HANDLING', 'Macro', 'api/macros/json_strict_nul_handling/index.html'); INSERT INTO searchIndex(name, type, path) VALUES ('JSON_THROW_USER', 'Macro', 'api/macros/json_throw_user/index.html'); INSERT INTO searchIndex(name, type, path) VALUES ('JSON_TRY_USER', 'Macro', 'api/macros/json_throw_user/index.html'); diff --git a/docs/mkdocs/docs/api/macros/json_strict_binary_utf8.md b/docs/mkdocs/docs/api/macros/json_strict_binary_utf8.md index ed7bbe6b3..4c573d56e 100644 --- a/docs/mkdocs/docs/api/macros/json_strict_binary_utf8.md +++ b/docs/mkdocs/docs/api/macros/json_strict_binary_utf8.md @@ -54,7 +54,7 @@ The default value is `0` (disabled, the behavior of version 3.12.0 and earlier i ## Examples -??? example "Default behavior (macro not defined)" +??? example "Example: default behavior (macro not defined)" Without the macro, the bytes are written unchanged: @@ -70,7 +70,7 @@ The default value is `0` (disabled, the behavior of version 3.12.0 and earlier i } ``` -??? example "Opt-in check (macro defined to 1)" +??? example "Example: opt-in check (macro defined to 1)" With the macro, ill-formed UTF-8 is rejected: diff --git a/docs/mkdocs/docs/api/macros/json_use_implicit_conversions.md b/docs/mkdocs/docs/api/macros/json_use_implicit_conversions.md index 11bf74b22..113344412 100644 --- a/docs/mkdocs/docs/api/macros/json_use_implicit_conversions.md +++ b/docs/mkdocs/docs/api/macros/json_use_implicit_conversions.md @@ -44,7 +44,7 @@ By default, implicit conversions are enabled. ## Examples -??? example +??? example "Example: implicit conversion" This is an example for an implicit conversion: @@ -61,7 +61,7 @@ By default, implicit conversions are enabled. auto s = j.get(); ``` -??? example "Conversion between `basic_json` specializations" +??? example "Example: conversion between `basic_json` specializations" A `basic_json` specialization with a different string type is also no longer converted implicitly when `JSON_USE_IMPLICIT_CONVERSIONS` is defined to `0`: diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index 1a78a6502..d6b45909a 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -110,15 +110,15 @@ class binary_reader @param[in] adapter input adapter to read from @param[in] format the binary format to parse - @param[in] error_handler how to treat text strings and object keys that + @param[in] error_handler_ how to treat text strings and object keys that are not well-formed UTF-8; none of the supported formats requires a decoder to reject those, so the default is to @ref error_handler_t::keep them unchanged, as every binary reader did before this parameter existed */ explicit binary_reader(InputAdapterType&& adapter, const input_format_t format = input_format_t::json, - const error_handler_t error_handler = error_handler_t::keep) noexcept - : ia(std::move(adapter)), input_format(format), error_handler(error_handler) + const error_handler_t error_handler_ = error_handler_t::keep) noexcept + : ia(std::move(adapter)), input_format(format), error_handler(error_handler_) { (void)detail::is_sax_static_asserts {}; } diff --git a/include/nlohmann/detail/json_pointer.hpp b/include/nlohmann/detail/json_pointer.hpp index 788af1047..30edb4a3b 100644 --- a/include/nlohmann/detail/json_pointer.hpp +++ b/include/nlohmann/detail/json_pointer.hpp @@ -322,6 +322,8 @@ class json_pointer typename BasicJsonType::size_type idx{}; switch (parse_array_index(s, idx)) { + // the branches differ in their messages, not after JSON_THROW's expansion + // NOLINTNEXTLINE(bugprone-branch-clone) case array_index_status::leading_zero: JSON_THROW(detail::parse_error::create(106, 0, detail::concat("array index '", s, "' must not begin with '0'"), nullptr)); case array_index_status::not_a_number: @@ -685,6 +687,8 @@ class json_pointer typename BasicJsonType::size_type idx{}; switch (parse_array_index(reference_token, idx)) { + // the branches differ in their messages, not after JSON_THROW's expansion + // NOLINTNEXTLINE(bugprone-branch-clone) case array_index_status::leading_zero: JSON_THROW(detail::parse_error::create(106, 0, detail::concat("array index '", reference_token, "' must not begin with '0'"), nullptr)); case array_index_status::not_a_number: diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index e74671db2..a4921f217 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -2260,18 +2260,18 @@ class binary_writer switch (error_handler) { case error_handler_t::keep: - return s; + return s; // NOLINT(bugprone-return-const-ref-from-parameter): callers pass lvalues that outlive the call case error_handler_t::strict: check_utf8(s, context); - return s; + return s; // NOLINT(bugprone-return-const-ref-from-parameter): callers pass lvalues that outlive the call case error_handler_t::replace: case error_handler_t::ignore: default: if (is_valid_utf8(s)) { - return s; + return s; // NOLINT(bugprone-return-const-ref-from-parameter): callers pass lvalues that outlive the call } storage = sanitize_utf8(s, error_handler); return storage; diff --git a/include/nlohmann/detail/output/serializer.hpp b/include/nlohmann/detail/output/serializer.hpp index dcaae8055..8306440ef 100644 --- a/include/nlohmann/detail/output/serializer.hpp +++ b/include/nlohmann/detail/output/serializer.hpp @@ -706,6 +706,11 @@ class serializer @a ensure_ascii is a template parameter here so that the branch on it is resolved once, outside the loop; see @ref dump_escaped. */ +#ifdef JSON_HEDLEY_MSVC_VERSION +#pragma warning(push) + // EnsureAscii is a template parameter; C++11 has no if constexpr +#pragma warning(disable : 4127) // conditional expression is constant +#endif template void dump_escaped_impl(const string_t& s) { @@ -1055,6 +1060,9 @@ class serializer } } } +#ifdef JSON_HEDLEY_MSVC_VERSION +#pragma warning(pop) +#endif private: /*! diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 421feda50..3f3645524 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -1048,6 +1048,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec never name it to call this constructor itself. */ basic_json(copy_construct_tag /*unused*/, const basic_json& src) + noexcept(std::is_nothrow_copy_constructible::value) : json_base_class_t(src) #if JSON_DIAGNOSTIC_POSITIONS , start_position(src.start_position) @@ -1442,7 +1443,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec be destroyed. */ template - void convert_iteratively(const BasicJsonType& val) + void convert_iteratively(const BasicJsonType& val, std::true_type /*unused*/) { using other_const_iterator = typename BasicJsonType::const_iterator; @@ -1536,21 +1537,38 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec if (JSON_HEDLEY_LIKELY(guard.okay())) { - // every element comes back to the converting constructor - if (val.is_object()) - { - using other_object_t = typename BasicJsonType::object_t; - JSONSerializer::to_json(*this, val.template get_ref()); - } - else - { - using other_array_t = typename BasicJsonType::array_t; - JSONSerializer::to_json(*this, val.template get_ref()); - } + convert_by_serializers(val); return; } - convert_iteratively(val); + // the iterative conversion needs to construct this object type's keys + // from those of @a val; if it cannot, neither can the range constructor, + // and the serializers convert @a val some other way (see #3425) + convert_iteratively(val, std::is_constructible {}); + } + + /// @brief convert the object or array @a val with the serializers; every + /// element comes back to the converting constructor + template + void convert_by_serializers(const BasicJsonType& val) + { + if (val.is_object()) + { + using other_object_t = typename BasicJsonType::object_t; + JSONSerializer::to_json(*this, val.template get_ref()); + } + else + { + using other_array_t = typename BasicJsonType::array_t; + JSONSerializer::to_json(*this, val.template get_ref()); + } + } + + /// @brief convert @a val whose keys cannot be converted, see @ref convert_structured + template + void convert_iteratively(const BasicJsonType& val, std::false_type /*unused*/) + { + convert_by_serializers(val); } @@ -1624,15 +1642,15 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec static compare_result compare_leaves(const_reference lhs, const_reference rhs, std::true_type /*ordered*/) noexcept { const std::partial_ordering order = lhs <=> rhs; // *NOPAD* - if (order == 0) + if (std::is_eq(order)) { return compare_result::equal; } - if (order < 0) + if (std::is_lt(order)) { return compare_result::less; } - if (order > 0) + if (std::is_gt(order)) { return compare_result::greater; } @@ -2704,6 +2722,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec { auto ret = ValueType(); JSONSerializer::from_json(*this, ret); + // false positive: ret is returned by value, not its address + // @infer-ignore STACK_VARIABLE_ADDRESS_ESCAPE return ret; } @@ -5058,7 +5078,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/operator_le/ template requires std::is_scalar_v - friend bool operator<=(ScalarType lhs, const_reference rhs) noexcept + friend bool operator<=(ScalarType lhs, const_reference rhs) noexcept(std::is_nothrow_constructible::value) { return basic_json(lhs) <= rhs; } @@ -5067,7 +5087,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/operator_ge/ template requires std::is_scalar_v - friend bool operator>=(ScalarType lhs, const_reference rhs) noexcept + friend bool operator>=(ScalarType lhs, const_reference rhs) noexcept(std::is_nothrow_constructible::value) { return basic_json(lhs) >= rhs; } diff --git a/include/nlohmann/ordered_map.hpp b/include/nlohmann/ordered_map.hpp index eb521114e..2e0970726 100644 --- a/include/nlohmann/ordered_map.hpp +++ b/include/nlohmann/ordered_map.hpp @@ -12,7 +12,7 @@ #include // equal_to, less #include // initializer_list #include // input_iterator_tag, iterator_traits -#include // allocator +#include // allocator // IWYU pragma: keep #include // for operator new (placement new) #include // for out_of_range #include // forward_as_tuple @@ -78,7 +78,7 @@ private: /// @brief find the entry for @a key, for either constness of @a self /// @note the single place that performs the linear key search template - static auto find_impl(Self& self, KeyType&& key) -> decltype(self.begin()) + static auto find_impl(Self& self, const KeyType& key) -> decltype(self.begin()) { for (auto it = self.begin(); it != self.end(); ++it) { diff --git a/nlohmann_json.natvis b/nlohmann_json.natvis index ed443145e..17029f1dd 100644 --- a/nlohmann_json.natvis +++ b/nlohmann_json.natvis @@ -455,6 +455,66 @@ + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + null @@ -755,6 +815,66 @@ + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + null @@ -995,6 +1115,66 @@ + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + null @@ -1175,6 +1355,66 @@ + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + null @@ -1295,6 +1535,66 @@ + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + null @@ -1355,6 +1655,126 @@ + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + null @@ -1595,6 +2015,66 @@ + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + null @@ -1775,6 +2255,66 @@ + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + null @@ -1895,6 +2435,66 @@ + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + null @@ -1955,6 +2555,126 @@ + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + null @@ -2135,6 +2855,66 @@ + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + null @@ -2255,6 +3035,66 @@ + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + null @@ -2315,6 +3155,126 @@ + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + null @@ -2435,6 +3395,66 @@ + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + null @@ -2495,6 +3515,126 @@ + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + null @@ -2555,6 +3695,186 @@ + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + null @@ -2735,6 +4055,66 @@ + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + null @@ -2855,6 +4235,66 @@ + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + null @@ -2915,6 +4355,126 @@ + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + null @@ -3035,6 +4595,66 @@ + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + null @@ -3095,6 +4715,126 @@ + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + null @@ -3155,6 +4895,186 @@ + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + null @@ -3275,6 +5195,66 @@ + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + null @@ -3335,6 +5315,126 @@ + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + null @@ -3395,6 +5495,186 @@ + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + null @@ -3455,6 +5735,246 @@ + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + null @@ -3575,6 +6095,66 @@ + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + null @@ -3635,6 +6215,126 @@ + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + null @@ -3695,6 +6395,186 @@ + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + null @@ -3755,6 +6635,246 @@ + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + null @@ -3815,6 +6935,306 @@ + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + null @@ -3875,4 +7295,424 @@ + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + + + + null + {*(m_data.m_value.object)} + {*(m_data.m_value.array)} + {*(m_data.m_value.string)} + {m_data.m_value.boolean} + {m_data.m_value.number_integer} + {m_data.m_value.number_unsigned} + {m_data.m_value.number_float} + discarded + + + *(m_data.m_value.object),view(simple) + + + *(m_data.m_value.array),view(simple) + + + + + + + {second} + + second + + + diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 511401699..75684866e 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -13696,15 +13696,15 @@ class binary_reader @param[in] adapter input adapter to read from @param[in] format the binary format to parse - @param[in] error_handler how to treat text strings and object keys that + @param[in] error_handler_ how to treat text strings and object keys that are not well-formed UTF-8; none of the supported formats requires a decoder to reject those, so the default is to @ref error_handler_t::keep them unchanged, as every binary reader did before this parameter existed */ explicit binary_reader(InputAdapterType&& adapter, const input_format_t format = input_format_t::json, - const error_handler_t error_handler = error_handler_t::keep) noexcept - : ia(std::move(adapter)), input_format(format), error_handler(error_handler) + const error_handler_t error_handler_ = error_handler_t::keep) noexcept + : ia(std::move(adapter)), input_format(format), error_handler(error_handler_) { (void)detail::is_sax_static_asserts {}; } @@ -19998,6 +19998,8 @@ class json_pointer typename BasicJsonType::size_type idx{}; switch (parse_array_index(s, idx)) { + // the branches differ in their messages, not after JSON_THROW's expansion + // NOLINTNEXTLINE(bugprone-branch-clone) case array_index_status::leading_zero: JSON_THROW(detail::parse_error::create(106, 0, detail::concat("array index '", s, "' must not begin with '0'"), nullptr)); case array_index_status::not_a_number: @@ -20361,6 +20363,8 @@ class json_pointer typename BasicJsonType::size_type idx{}; switch (parse_array_index(reference_token, idx)) { + // the branches differ in their messages, not after JSON_THROW's expansion + // NOLINTNEXTLINE(bugprone-branch-clone) case array_index_status::leading_zero: JSON_THROW(detail::parse_error::create(106, 0, detail::concat("array index '", reference_token, "' must not begin with '0'"), nullptr)); case array_index_status::not_a_number: @@ -23485,18 +23489,18 @@ class binary_writer switch (error_handler) { case error_handler_t::keep: - return s; + return s; // NOLINT(bugprone-return-const-ref-from-parameter): callers pass lvalues that outlive the call case error_handler_t::strict: check_utf8(s, context); - return s; + return s; // NOLINT(bugprone-return-const-ref-from-parameter): callers pass lvalues that outlive the call case error_handler_t::replace: case error_handler_t::ignore: default: if (is_valid_utf8(s)) { - return s; + return s; // NOLINT(bugprone-return-const-ref-from-parameter): callers pass lvalues that outlive the call } storage = sanitize_utf8(s, error_handler); return storage; @@ -25681,6 +25685,11 @@ class serializer @a ensure_ascii is a template parameter here so that the branch on it is resolved once, outside the loop; see @ref dump_escaped. */ +#ifdef JSON_HEDLEY_MSVC_VERSION +#pragma warning(push) + // EnsureAscii is a template parameter; C++11 has no if constexpr +#pragma warning(disable : 4127) // conditional expression is constant +#endif template void dump_escaped_impl(const string_t& s) { @@ -26030,6 +26039,9 @@ class serializer } } } +#ifdef JSON_HEDLEY_MSVC_VERSION +#pragma warning(pop) +#endif private: /*! @@ -26612,7 +26624,7 @@ NLOHMANN_JSON_NAMESPACE_END #include // equal_to, less #include // initializer_list #include // input_iterator_tag, iterator_traits -#include // allocator +#include // allocator // IWYU pragma: keep #include // for operator new (placement new) #include // for out_of_range #include // forward_as_tuple @@ -26681,7 +26693,7 @@ private: /// @brief find the entry for @a key, for either constness of @a self /// @note the single place that performs the linear key search template - static auto find_impl(Self& self, KeyType&& key) -> decltype(self.begin()) + static auto find_impl(Self& self, const KeyType& key) -> decltype(self.begin()) { for (auto it = self.begin(); it != self.end(); ++it) { @@ -27985,6 +27997,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec never name it to call this constructor itself. */ basic_json(copy_construct_tag /*unused*/, const basic_json& src) + noexcept(std::is_nothrow_copy_constructible::value) : json_base_class_t(src) #if JSON_DIAGNOSTIC_POSITIONS , start_position(src.start_position) @@ -28379,7 +28392,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec be destroyed. */ template - void convert_iteratively(const BasicJsonType& val) + void convert_iteratively(const BasicJsonType& val, std::true_type /*unused*/) { using other_const_iterator = typename BasicJsonType::const_iterator; @@ -28473,21 +28486,38 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec if (JSON_HEDLEY_LIKELY(guard.okay())) { - // every element comes back to the converting constructor - if (val.is_object()) - { - using other_object_t = typename BasicJsonType::object_t; - JSONSerializer::to_json(*this, val.template get_ref()); - } - else - { - using other_array_t = typename BasicJsonType::array_t; - JSONSerializer::to_json(*this, val.template get_ref()); - } + convert_by_serializers(val); return; } - convert_iteratively(val); + // the iterative conversion needs to construct this object type's keys + // from those of @a val; if it cannot, neither can the range constructor, + // and the serializers convert @a val some other way (see #3425) + convert_iteratively(val, std::is_constructible {}); + } + + /// @brief convert the object or array @a val with the serializers; every + /// element comes back to the converting constructor + template + void convert_by_serializers(const BasicJsonType& val) + { + if (val.is_object()) + { + using other_object_t = typename BasicJsonType::object_t; + JSONSerializer::to_json(*this, val.template get_ref()); + } + else + { + using other_array_t = typename BasicJsonType::array_t; + JSONSerializer::to_json(*this, val.template get_ref()); + } + } + + /// @brief convert @a val whose keys cannot be converted, see @ref convert_structured + template + void convert_iteratively(const BasicJsonType& val, std::false_type /*unused*/) + { + convert_by_serializers(val); } @@ -28561,15 +28591,15 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec static compare_result compare_leaves(const_reference lhs, const_reference rhs, std::true_type /*ordered*/) noexcept { const std::partial_ordering order = lhs <=> rhs; // *NOPAD* - if (order == 0) + if (std::is_eq(order)) { return compare_result::equal; } - if (order < 0) + if (std::is_lt(order)) { return compare_result::less; } - if (order > 0) + if (std::is_gt(order)) { return compare_result::greater; } @@ -29641,6 +29671,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec { auto ret = ValueType(); JSONSerializer::from_json(*this, ret); + // false positive: ret is returned by value, not its address + // @infer-ignore STACK_VARIABLE_ADDRESS_ESCAPE return ret; } @@ -31995,7 +32027,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/operator_le/ template requires std::is_scalar_v - friend bool operator<=(ScalarType lhs, const_reference rhs) noexcept + friend bool operator<=(ScalarType lhs, const_reference rhs) noexcept(std::is_nothrow_constructible::value) { return basic_json(lhs) <= rhs; } @@ -32004,7 +32036,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/operator_ge/ template requires std::is_scalar_v - friend bool operator>=(ScalarType lhs, const_reference rhs) noexcept + friend bool operator>=(ScalarType lhs, const_reference rhs) noexcept(std::is_nothrow_constructible::value) { return basic_json(lhs) >= rhs; } diff --git a/tests/src/test_utils.hpp b/tests/src/test_utils.hpp index b2a381fb0..8f7bd06a0 100644 --- a/tests/src/test_utils.hpp +++ b/tests/src/test_utils.hpp @@ -10,7 +10,8 @@ #include // uint8_t #include // size_t -#include // ifstream, istreambuf_iterator, ios +#include // ifstream, ios +#include // istream_iterator #include // vector namespace utils diff --git a/tests/src/unit-allocator.cpp b/tests/src/unit-allocator.cpp index fbdfaa351..bc283ab28 100644 --- a/tests/src/unit-allocator.cpp +++ b/tests/src/unit-allocator.cpp @@ -567,7 +567,7 @@ struct allocator_no_forward : std::allocator { allocator_no_forward() = default; template - allocator_no_forward(allocator_no_forward /*unused*/) {} + allocator_no_forward(const allocator_no_forward& /*unused*/) {} template struct rebind diff --git a/tests/src/unit-binary_utf8_error_handler.cpp b/tests/src/unit-binary_utf8_error_handler.cpp index 85505bffb..c3f2435fd 100644 --- a/tests/src/unit-binary_utf8_error_handler.cpp +++ b/tests/src/unit-binary_utf8_error_handler.cpp @@ -7,6 +7,7 @@ // SPDX-License-Identifier: MIT #include "doctest_compatibility.h" +#include "test_utils.hpp" #include using nlohmann::json; @@ -25,18 +26,27 @@ struct ill_formed_case // RFC 3629 ill-formed sequences used throughout this file, plus one // well-formed sequence for contrast -const std::vector ill_formed_cases = +std::vector ill_formed_cases() { - {"overlong", "\xC0\xAE"}, - {"lone_0xFF", "\xFF"}, - {"truncated", "\xE2\x82"}, - {"surrogate", "\xED\xA0\x80"}, -}; + return + { + {"overlong", "\xC0\xAE"}, + {"lone_0xFF", "\xFF"}, + {"truncated", "\xE2\x82"}, + {"surrogate", "\xED\xA0\x80"}, + }; +} -const std::string valid_sequence = "\xC3\xA9"; // U+00E9, "é" +std::string valid_sequence() +{ + return "\xC3\xA9"; // U+00E9, "é" +} using eh = json::error_handler_t; -const std::vector all_handlers = {eh::strict, eh::replace, eh::ignore, eh::keep}; +std::vector all_handlers() +{ + return {eh::strict, eh::replace, eh::ignore, eh::keep}; +} // what dump()+parse() produces for a sanitizing error_handler; this is the // ground truth every binary writer/reader is checked against @@ -51,9 +61,9 @@ TEST_CASE("UTF-8 error_handler for the binary readers and writers") { SECTION("writers: string value") { - for (const auto& c : ill_formed_cases) + for (const auto& c : ill_formed_cases()) { - CAPTURE(c.name); + CAPTURE(c.name) const json jval = c.bytes; CHECK_THROWS_AS(json::to_cbor(jval, eh::strict), json::type_error&); @@ -71,7 +81,7 @@ TEST_CASE("UTF-8 error_handler for the binary readers and writers") eh::replace, eh::ignore }) { - CAPTURE(static_cast(h)); + CAPTURE(static_cast(h)) const std::string expected = dump_and_parse(c.bytes, h); CHECK(json::from_cbor(json::to_cbor(jval, h)).get() == expected); @@ -103,9 +113,9 @@ TEST_CASE("UTF-8 error_handler for the binary readers and writers") SECTION("writers: object key") { - for (const auto& c : ill_formed_cases) + for (const auto& c : ill_formed_cases()) { - CAPTURE(c.name); + CAPTURE(c.name) json jobj; jobj[c.bytes] = 1; @@ -120,7 +130,7 @@ TEST_CASE("UTF-8 error_handler for the binary readers and writers") eh::replace, eh::ignore }) { - CAPTURE(static_cast(h)); + CAPTURE(static_cast(h)) const std::string expected = dump_and_parse(c.bytes, h); CHECK(json::from_cbor(json::to_cbor(jobj, h)).begin().key() == expected); @@ -141,9 +151,9 @@ TEST_CASE("UTF-8 error_handler for the binary readers and writers") SECTION("readers: string value") { - for (const auto& c : ill_formed_cases) + for (const auto& c : ill_formed_cases()) { - CAPTURE(c.name); + CAPTURE(c.name) // bytes produced the lenient (keep) way, as any binary reader // accepted them before this parameter existed @@ -166,15 +176,15 @@ TEST_CASE("UTF-8 error_handler for the binary readers and writers") CHECK(json::from_bson(bson_bytes)["k"].get() == c.bytes); // strict: parse_error.113, discarded (not thrown) when allow_exceptions is false - CHECK_THROWS_AS(json::from_cbor(cbor_bytes, true, true, json::cbor_tag_handler_t::error, eh::strict), json::parse_error&); + CHECK_THROWS_AS(utils::ignore_return_value(json::from_cbor(cbor_bytes, true, true, json::cbor_tag_handler_t::error, eh::strict)), json::parse_error&); CHECK(json::from_cbor(cbor_bytes, true, false, json::cbor_tag_handler_t::error, eh::strict).is_discarded()); - CHECK_THROWS_AS(json::from_msgpack(msgpack_bytes, true, true, eh::strict), json::parse_error&); + CHECK_THROWS_AS(utils::ignore_return_value(json::from_msgpack(msgpack_bytes, true, true, eh::strict)), json::parse_error&); CHECK(json::from_msgpack(msgpack_bytes, true, false, eh::strict).is_discarded()); - CHECK_THROWS_AS(json::from_ubjson(ubjson_bytes, true, true, eh::strict), json::parse_error&); + CHECK_THROWS_AS(utils::ignore_return_value(json::from_ubjson(ubjson_bytes, true, true, eh::strict)), json::parse_error&); CHECK(json::from_ubjson(ubjson_bytes, true, false, eh::strict).is_discarded()); - CHECK_THROWS_AS(json::from_bjdata(bjdata_bytes, true, true, eh::strict), json::parse_error&); + CHECK_THROWS_AS(utils::ignore_return_value(json::from_bjdata(bjdata_bytes, true, true, eh::strict)), json::parse_error&); CHECK(json::from_bjdata(bjdata_bytes, true, false, eh::strict).is_discarded()); - CHECK_THROWS_AS(json::from_bson(bson_bytes, true, true, eh::strict), json::parse_error&); + CHECK_THROWS_AS(utils::ignore_return_value(json::from_bson(bson_bytes, true, true, eh::strict)), json::parse_error&); CHECK(json::from_bson(bson_bytes, true, false, eh::strict).is_discarded()); // replace / ignore: match what dump() would have sanitized the same bytes to @@ -183,7 +193,7 @@ TEST_CASE("UTF-8 error_handler for the binary readers and writers") eh::replace, eh::ignore }) { - CAPTURE(static_cast(h)); + CAPTURE(static_cast(h)) const std::string expected = dump_and_parse(c.bytes, h); CHECK(json::from_cbor(cbor_bytes, true, true, json::cbor_tag_handler_t::error, h).get() == expected); @@ -197,9 +207,9 @@ TEST_CASE("UTF-8 error_handler for the binary readers and writers") SECTION("readers: object key") { - for (const auto& c : ill_formed_cases) + for (const auto& c : ill_formed_cases()) { - CAPTURE(c.name); + CAPTURE(c.name) json jobj; jobj[c.bytes] = 1; @@ -215,18 +225,18 @@ TEST_CASE("UTF-8 error_handler for the binary readers and writers") CHECK(json::from_bjdata(bjdata_bytes).begin().key() == c.bytes); CHECK(json::from_bson(bson_bytes).begin().key() == c.bytes); - CHECK_THROWS_AS(json::from_cbor(cbor_bytes, true, true, json::cbor_tag_handler_t::error, eh::strict), json::parse_error&); - CHECK_THROWS_AS(json::from_msgpack(msgpack_bytes, true, true, eh::strict), json::parse_error&); - CHECK_THROWS_AS(json::from_ubjson(ubjson_bytes, true, true, eh::strict), json::parse_error&); - CHECK_THROWS_AS(json::from_bjdata(bjdata_bytes, true, true, eh::strict), json::parse_error&); - CHECK_THROWS_AS(json::from_bson(bson_bytes, true, true, eh::strict), json::parse_error&); + CHECK_THROWS_AS(utils::ignore_return_value(json::from_cbor(cbor_bytes, true, true, json::cbor_tag_handler_t::error, eh::strict)), json::parse_error&); + CHECK_THROWS_AS(utils::ignore_return_value(json::from_msgpack(msgpack_bytes, true, true, eh::strict)), json::parse_error&); + CHECK_THROWS_AS(utils::ignore_return_value(json::from_ubjson(ubjson_bytes, true, true, eh::strict)), json::parse_error&); + CHECK_THROWS_AS(utils::ignore_return_value(json::from_bjdata(bjdata_bytes, true, true, eh::strict)), json::parse_error&); + CHECK_THROWS_AS(utils::ignore_return_value(json::from_bson(bson_bytes, true, true, eh::strict)), json::parse_error&); for (const auto h : { eh::replace, eh::ignore }) { - CAPTURE(static_cast(h)); + CAPTURE(static_cast(h)) const std::string expected = dump_and_parse(c.bytes, h); CHECK(json::from_cbor(cbor_bytes, true, true, json::cbor_tag_handler_t::error, h).begin().key() == expected); @@ -240,30 +250,30 @@ TEST_CASE("UTF-8 error_handler for the binary readers and writers") SECTION("well-formed UTF-8 is unaffected by error_handler") { - const json jval = valid_sequence; + const json jval = valid_sequence(); json jobj; - jobj[valid_sequence] = valid_sequence; + jobj[valid_sequence()] = valid_sequence(); - for (const auto h : all_handlers) + for (const auto h : all_handlers()) { - CAPTURE(static_cast(h)); + CAPTURE(static_cast(h)) - CHECK(json::from_cbor(json::to_cbor(jval, h)).get() == valid_sequence); - CHECK(json::from_msgpack(json::to_msgpack(jval, h)).get() == valid_sequence); - CHECK(json::from_ubjson(json::to_ubjson(jval, false, false, h)).get() == valid_sequence); - CHECK(json::from_bjdata(json::to_bjdata(jval, false, false, json::bjdata_version_t::draft2, h)).get() == valid_sequence); - CHECK(json::from_bson(json::to_bson(jobj, h)).begin().key() == valid_sequence); + CHECK(json::from_cbor(json::to_cbor(jval, h)).get() == valid_sequence()); + CHECK(json::from_msgpack(json::to_msgpack(jval, h)).get() == valid_sequence()); + CHECK(json::from_ubjson(json::to_ubjson(jval, false, false, h)).get() == valid_sequence()); + CHECK(json::from_bjdata(json::to_bjdata(jval, false, false, json::bjdata_version_t::draft2, h)).get() == valid_sequence()); + CHECK(json::from_bson(json::to_bson(jobj, h)).begin().key() == valid_sequence()); - CHECK(json::from_cbor(json::to_cbor(jval, eh::keep), true, true, json::cbor_tag_handler_t::error, h).get() == valid_sequence); - CHECK(json::from_msgpack(json::to_msgpack(jval), true, true, h).get() == valid_sequence); + CHECK(json::from_cbor(json::to_cbor(jval, eh::keep), true, true, json::cbor_tag_handler_t::error, h).get() == valid_sequence()); + CHECK(json::from_msgpack(json::to_msgpack(jval), true, true, h).get() == valid_sequence()); } } SECTION("dump() with error_handler_t::keep writes raw bytes as is") { - for (const auto& c : ill_formed_cases) + for (const auto& c : ill_formed_cases()) { - CAPTURE(c.name); + CAPTURE(c.name) const json jval = c.bytes; const std::string dumped = jval.dump(-1, ' ', false, eh::keep); @@ -276,10 +286,10 @@ TEST_CASE("UTF-8 error_handler for the binary readers and writers") // well-formed characters around an ill-formed sequence are still // escaped as usual under ensure_ascii - const json mixed = valid_sequence + ill_formed_cases[1].bytes; // "é" + lone 0xFF + const json mixed = valid_sequence() + ill_formed_cases()[1].bytes; // "é" + lone 0xFF const std::string dumped_mixed = mixed.dump(-1, ' ', true, eh::keep); CHECK(dumped_mixed.find("\\u00e9") != std::string::npos); - CHECK(dumped_mixed.find(ill_formed_cases[1].bytes) != std::string::npos); + CHECK(dumped_mixed.find(ill_formed_cases()[1].bytes) != std::string::npos); // the byte that ends an ill-formed sequence is read again, so a quote, // a backslash, or a control character after it is still escaped, and @@ -289,7 +299,7 @@ TEST_CASE("UTF-8 error_handler for the binary readers and writers") false, true }) { - CAPTURE(ensure_ascii); + CAPTURE(ensure_ascii) CHECK(json("\xC3\"").dump(-1, ' ', ensure_ascii, eh::keep) == "\"\xC3\\\"\""); CHECK(json("\xC3\\").dump(-1, ' ', ensure_ascii, eh::keep) == "\"\xC3\\\\\""); CHECK(json("\xC3\n").dump(-1, ' ', ensure_ascii, eh::keep) == "\"\xC3\\n\""); @@ -303,12 +313,12 @@ TEST_CASE("UTF-8 error_handler for the binary readers and writers") SECTION("to_msgpack defaults to keep; to_bon8 is not affected by error_handler") { - const json jval = ill_formed_cases[1].bytes; // lone 0xFF + const json jval = ill_formed_cases()[1].bytes; // lone 0xFF // to_msgpack's error_handler defaults to keep, as MessagePack's spec // allows any bytes in a str, so the bytes are passed through CHECK(json::to_msgpack(jval) == json::to_msgpack(jval, eh::keep)); - CHECK(json::from_msgpack(json::to_msgpack(jval)).get() == ill_formed_cases[1].bytes); + CHECK(json::from_msgpack(json::to_msgpack(jval)).get() == ill_formed_cases()[1].bytes); // the diagnostics context of an ill-formed key is the object json jobj; @@ -322,14 +332,14 @@ TEST_CASE("UTF-8 error_handler for the binary readers and writers") SECTION("allow_exceptions=false with error_handler_t::strict discards the value") { - const auto bytes = json::to_cbor(json(ill_formed_cases[0].bytes), eh::keep); + const auto bytes = json::to_cbor(json(ill_formed_cases()[0].bytes), eh::keep); const json result = json::from_cbor(bytes, true, false, json::cbor_tag_handler_t::error, eh::strict); CHECK(result.is_discarded()); } SECTION("default parameters are unchanged") { - const json jval = ill_formed_cases[0].bytes; + const json jval = ill_formed_cases()[0].bytes; // to_*: the default error_handler is keep, so ill-formed bytes are // written unchanged, exactly as in release 3.12.0 (it is strict only @@ -347,16 +357,16 @@ TEST_CASE("UTF-8 error_handler for the binary readers and writers") // from_*: the default error_handler is keep, so ill-formed bytes are // still accepted unchanged, exactly as in release 3.12.0 const auto cbor_bytes = json::to_cbor(jval, eh::keep); - CHECK(json::from_cbor(cbor_bytes).get() == ill_formed_cases[0].bytes); + CHECK(json::from_cbor(cbor_bytes).get() == ill_formed_cases()[0].bytes); const auto ubjson_bytes = json::to_ubjson(jval, false, false, eh::keep); - CHECK(json::from_ubjson(ubjson_bytes).get() == ill_formed_cases[0].bytes); + CHECK(json::from_ubjson(ubjson_bytes).get() == ill_formed_cases()[0].bytes); const auto bjdata_bytes = json::to_bjdata(jval, false, false, json::bjdata_version_t::draft2, eh::keep); - CHECK(json::from_bjdata(bjdata_bytes).get() == ill_formed_cases[0].bytes); + CHECK(json::from_bjdata(bjdata_bytes).get() == ill_formed_cases()[0].bytes); const auto msgpack_bytes = json::to_msgpack(jval); - CHECK(json::from_msgpack(msgpack_bytes).get() == ill_formed_cases[0].bytes); + CHECK(json::from_msgpack(msgpack_bytes).get() == ill_formed_cases()[0].bytes); json bson_obj; bson_obj["k"] = jval; const auto bson_bytes = json::to_bson(bson_obj, eh::keep); - CHECK(json::from_bson(bson_bytes)["k"].get() == ill_formed_cases[0].bytes); + CHECK(json::from_bson(bson_bytes)["k"].get() == ill_formed_cases()[0].bytes); } } diff --git a/tests/src/unit-binary_utf8_strict.cpp b/tests/src/unit-binary_utf8_strict.cpp index 2ac8aabf4..96d96e61c 100644 --- a/tests/src/unit-binary_utf8_strict.cpp +++ b/tests/src/unit-binary_utf8_strict.cpp @@ -83,21 +83,45 @@ TEST_CASE("JSON_STRICT_BINARY_UTF8 (see #5529, #5651)") // any bytes reach the output adapter (the BSON document length // prefix must be known up front, so nothing is written incrementally) std::vector out{0x42}; // a sentinel byte the writer must not touch - CHECK_THROWS_WITH_AS(json::to_bson(json{{"s", "\xFF"}}, nlohmann::detail::output_adapter(out)), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xFF", json::type_error&); +#if JSON_DIAGNOSTICS + CHECK_THROWS_WITH_AS(json::to_bson(json {{"s", "\xFF"}}, nlohmann::detail::output_adapter(out)), "[json.exception.type_error.316] (/s) invalid UTF-8 byte at index 0: 0xFF", json::type_error&); +#else + CHECK_THROWS_WITH_AS(json::to_bson(json {{"s", "\xFF"}}, nlohmann::detail::output_adapter(out)), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xFF", json::type_error&); +#endif CHECK(out == std::vector {0x42}); - CHECK_THROWS_WITH_AS(json::to_bson(json{{"s", "\xFF"}}), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xFF", json::type_error&); +#if JSON_DIAGNOSTICS + CHECK_THROWS_WITH_AS(json::to_bson(json {{"s", "\xFF"}}), "[json.exception.type_error.316] (/s) invalid UTF-8 byte at index 0: 0xFF", json::type_error&); +#else + CHECK_THROWS_WITH_AS(json::to_bson(json {{"s", "\xFF"}}), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xFF", json::type_error&); +#endif // a truncated multi-byte sequence - CHECK_THROWS_WITH_AS(json::to_bson(json{{"s", "\xC3"}}), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xC3", json::type_error&); +#if JSON_DIAGNOSTICS + CHECK_THROWS_WITH_AS(json::to_bson(json {{"s", "\xC3"}}), "[json.exception.type_error.316] (/s) invalid UTF-8 byte at index 0: 0xC3", json::type_error&); +#else + CHECK_THROWS_WITH_AS(json::to_bson(json {{"s", "\xC3"}}), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xC3", json::type_error&); +#endif // an encoded surrogate half (U+D800) - CHECK_THROWS_WITH_AS(json::to_bson(json{{"s", "\xED\xA0\x80"}}), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xED", json::type_error&); +#if JSON_DIAGNOSTICS + CHECK_THROWS_WITH_AS(json::to_bson(json {{"s", "\xED\xA0\x80"}}), "[json.exception.type_error.316] (/s) invalid UTF-8 byte at index 0: 0xED", json::type_error&); +#else + CHECK_THROWS_WITH_AS(json::to_bson(json {{"s", "\xED\xA0\x80"}}), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xED", json::type_error&); +#endif // an overlong encoding of '.' - CHECK_THROWS_WITH_AS(json::to_bson(json{{"s", "\xC0\xAF"}}), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xC0", json::type_error&); +#if JSON_DIAGNOSTICS + CHECK_THROWS_WITH_AS(json::to_bson(json {{"s", "\xC0\xAF"}}), "[json.exception.type_error.316] (/s) invalid UTF-8 byte at index 0: 0xC0", json::type_error&); +#else + CHECK_THROWS_WITH_AS(json::to_bson(json {{"s", "\xC0\xAF"}}), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xC0", json::type_error&); +#endif // an object key with ill-formed UTF-8 is rejected as well; unlike // the reader (which never validates element names), the writer // checks both string values and object keys - CHECK_THROWS_WITH_AS(json::to_bson(json{{"\xFF", 1}}), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xFF", json::type_error&); +#if JSON_DIAGNOSTICS + CHECK_THROWS_WITH_AS(json::to_bson(json {{"\xFF", 1}}), "[json.exception.type_error.316] (/\xFF) invalid UTF-8 byte at index 0: 0xFF", json::type_error&); +#else + CHECK_THROWS_WITH_AS(json::to_bson(json {{"\xFF", 1}}), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xFF", json::type_error&); +#endif } SECTION("an explicit error_handler overrides the default") diff --git a/tests/src/unit-bjdata.cpp b/tests/src/unit-bjdata.cpp index f34df1717..0098541cf 100644 --- a/tests/src/unit-bjdata.cpp +++ b/tests/src/unit-bjdata.cpp @@ -3921,7 +3921,7 @@ TEST_CASE("Universal Binary JSON Specification Examples 1") CHECK_NOTHROW(j = json::from_bjdata(v)); REQUIRE(j.is_string()); CHECK(j.get_ref() == std::string("\xc0\xae")); - CHECK_THROWS_AS(j.dump(), json::type_error&); + CHECK_THROWS_AS(utils::ignore_return_value(j.dump()), json::type_error&); CHECK(json::from_bjdata(json::to_bjdata(j)) == j); // the same bytes as an object key round-trip as well diff --git a/tests/src/unit-bon8.cpp b/tests/src/unit-bon8.cpp index a751d7dbe..491189867 100644 --- a/tests/src/unit-bon8.cpp +++ b/tests/src/unit-bon8.cpp @@ -786,6 +786,7 @@ TEST_CASE("Parse BON8 directly from a file using iterator and sentinel") CHECK((parsed.is_object() || parsed.is_array())); } +#if !defined(JSON_NOEXCEPTION) // corpus values that do not survive the round trip are skipped by catching the exception TEST_CASE("BON8 round-trip invariants") { // This checks what the parse_bon8_fuzzer driver checks (see @@ -818,6 +819,7 @@ TEST_CASE("BON8 round-trip invariants") CHECK(json::to_bon8(j2) == vec); } } +#endif TEST_CASE("BON8 roundtrips" * doctest::skip()) { diff --git a/tests/src/unit-bson.cpp b/tests/src/unit-bson.cpp index 92a14e6fe..17819dc65 100644 --- a/tests/src/unit-bson.cpp +++ b/tests/src/unit-bson.cpp @@ -62,6 +62,8 @@ class huge_string_t : public std::string { public: using std::string::string; + // inheriting std::string's constructors does not inherit its default constructor + huge_string_t() = default; huge_string_t(const std::string& s) : std::string(s) {} // NOLINT(google-explicit-constructor,hicpp-explicit-conversions) // returns a copy of @a s whose size() pretends to be huge @@ -174,7 +176,7 @@ TEST_CASE("BSON") REQUIRE(j.contains("s")); CHECK(j["s"].get_ref() == std::string("\xc0\xae")); // dump() still requires valid UTF-8 and throws for such a value - CHECK_THROWS_AS(j.dump(), json::type_error&); + CHECK_THROWS_AS(utils::ignore_return_value(j.dump()), json::type_error&); // to_bson() writes the bytes back unchanged, as before 3.13.0, // unless JSON_STRICT_BINARY_UTF8 is enabled (see unit-binary_utf8_strict.cpp) CHECK(json::from_bson(json::to_bson(j)) == j); diff --git a/tests/src/unit-cbor.cpp b/tests/src/unit-cbor.cpp index 633f50fea..f3f4301e5 100644 --- a/tests/src/unit-cbor.cpp +++ b/tests/src/unit-cbor.cpp @@ -1820,7 +1820,7 @@ TEST_CASE("CBOR") // dump() still requires valid UTF-8 and throws for such a value, // unless an error handler that replaces or ignores the bytes is // passed - CHECK_THROWS_AS(j_value.dump(), json::type_error&); + CHECK_THROWS_AS(utils::ignore_return_value(j_value.dump()), json::type_error&); // to_cbor() writes the bytes back unchanged, as before 3.13.0, // unless JSON_STRICT_BINARY_UTF8 is enabled (see unit-binary_utf8_strict.cpp) CHECK(json::from_cbor(json::to_cbor(j_value)) == j_value); @@ -1878,13 +1878,13 @@ TEST_CASE("CBOR") // a truncated code point is kept as is CHECK_NOTHROW(_ = json::from_cbor(std::vector({0x7f, 0x61, 0xc3, 0xff}))); CHECK(_ == "\xc3"); - CHECK_THROWS_AS(_.dump(), json::type_error&); + CHECK_THROWS_AS(utils::ignore_return_value(_.dump()), json::type_error&); CHECK(json::from_cbor(json::to_cbor(_)) == _); // an ill-formed later chunk is kept after valid ones CHECK_NOTHROW(_ = json::from_cbor(std::vector({0x7f, 0x62, 0xc3, 0xa9, 0x62, 0xc0, 0xae, 0xff}))); CHECK(_ == "\xc3\xa9\xc0\xae"); - CHECK_THROWS_AS(_.dump(), json::type_error&); + CHECK_THROWS_AS(utils::ignore_return_value(_.dump()), json::type_error&); // valid multi-byte chunks are accepted CHECK(json::from_cbor(std::vector({0x7f, 0x62, 0xc3, 0xa9, 0x62, 0xc3, 0xb6, 0xff})) == "\xc3\xa9\xc3\xb6"); @@ -2390,6 +2390,7 @@ TEST_CASE("issue #5405 - array reserve for definite-length CBOR arrays") } } +#if !defined(JSON_NOEXCEPTION) // corpus values that do not survive the round trip are skipped by catching the exception TEST_CASE("CBOR round-trip invariants") { // This checks what the parse_cbor_fuzzer driver checks (see @@ -2422,6 +2423,7 @@ TEST_CASE("CBOR round-trip invariants") CHECK(json::to_cbor(j2) == vec); } } +#endif TEST_CASE("CBOR roundtrips" * doctest::skip()) { diff --git a/tests/src/unit-class_parser.cpp b/tests/src/unit-class_parser.cpp index 0ae5e382b..f2de97ea3 100644 --- a/tests/src/unit-class_parser.cpp +++ b/tests/src/unit-class_parser.cpp @@ -2704,9 +2704,9 @@ TEST_CASE("diagnostic positions: value lifetime, input adapters, and SAX") CHECK(b["b"].end_pos() == nested_end); // the moved-from value is reset to a null and reports npos - CHECK(a.is_null()); // NOLINT(bugprone-use-after-move,clang-analyzer-cplusplus.Move) - CHECK(a.start_pos() == std::string::npos); // NOLINT(bugprone-use-after-move,clang-analyzer-cplusplus.Move) - CHECK(a.end_pos() == std::string::npos); // NOLINT(bugprone-use-after-move,clang-analyzer-cplusplus.Move) + CHECK(a.is_null()); // NOLINT(bugprone-use-after-move,hicpp-invalid-access-moved,clang-analyzer-cplusplus.Move) + CHECK(a.start_pos() == std::string::npos); // NOLINT(bugprone-use-after-move,hicpp-invalid-access-moved,clang-analyzer-cplusplus.Move) + CHECK(a.end_pos() == std::string::npos); // NOLINT(bugprone-use-after-move,hicpp-invalid-access-moved,clang-analyzer-cplusplus.Move) } SECTION("swap() exchanges positions along with values") diff --git a/tests/src/unit-comparison.cpp b/tests/src/unit-comparison.cpp index 15715fec2..b9257cb17 100644 --- a/tests/src/unit-comparison.cpp +++ b/tests/src/unit-comparison.cpp @@ -969,7 +969,7 @@ TEST_CASE("copying an object preserves its comparator's state") for (const std::size_t depth : std::vector {0, 127, 128, 200}) { - CAPTURE(depth); + CAPTURE(depth) key_case_json original = object; for (std::size_t i = 0; i < depth; ++i) @@ -1138,7 +1138,7 @@ TEST_CASE("operator<=> of binary values with a different subtype does not depend // and must still agree with the levels that do for (const std::size_t depth : std::vector {0, 127, 128, 200}) { - CAPTURE(depth); + CAPTURE(depth) const json x = deep(a, depth); const json y = deep(b, depth); CHECK((x <=> y) == std::partial_ordering::less); // *NOPAD* diff --git a/tests/src/unit-conversions.cpp b/tests/src/unit-conversions1.cpp similarity index 52% rename from tests/src/unit-conversions.cpp rename to tests/src/unit-conversions1.cpp index 5f83a9253..3d39e10c9 100644 --- a/tests/src/unit-conversions.cpp +++ b/tests/src/unit-conversions1.cpp @@ -15,11 +15,6 @@ #include "doctest_compatibility.h" -// skip tests if JSON_DisableEnumSerialization=ON (#4384) -#if defined(JSON_DISABLE_ENUM_SERIALIZATION) && (JSON_DISABLE_ENUM_SERIALIZATION == 1) - #define SKIP_TESTS_FOR_ENUM_SERIALIZATION -#endif - #define JSON_TESTS_PRIVATE #include using nlohmann::json; @@ -1259,808 +1254,8 @@ TEST_CASE("value conversion") } } #endif - - SECTION("get a binary value (explicit)") - { - json::binary_t const n_reference{{1, 2, 3}}; - json j(n_reference); - - SECTION("binary_t") - { - json::binary_t const b = j.get(); - CHECK(*json(b).m_data.m_value.binary == *j.m_data.m_value.binary); - } - - SECTION("get_binary()") - { - SECTION("non-const") - { - auto& b = j.get_binary(); - CHECK(*json(b).m_data.m_value.binary == *j.m_data.m_value.binary); - } - - SECTION("non-const") - { - const json j_const = j; // NOLINT(performance-unnecessary-copy-initialization) - const auto& b = j_const.get_binary(); - CHECK(*json(b).m_data.m_value.binary == *j.m_data.m_value.binary); - } - } - - SECTION("exception in case of a non-string type") - { - json j_null(json::value_t::null); - json j_object(json::value_t::object); - json j_array(json::value_t::array); - json j_string(json::value_t::string); - json j_boolean(json::value_t::boolean); - const json j_null_const(json::value_t::null); - const json j_object_const(json::value_t::object); - const json j_array_const(json::value_t::array); - const json j_string_const(json::value_t::string); - const json j_boolean_const(json::value_t::boolean); - - CHECK_THROWS_WITH_AS(j_null.get(), - "[json.exception.type_error.302] type must be binary, but is null", - json::type_error&); - CHECK_THROWS_WITH_AS(j_object.get(), - "[json.exception.type_error.302] type must be binary, but is object", - json::type_error&); - CHECK_THROWS_WITH_AS(j_array.get(), - "[json.exception.type_error.302] type must be binary, but is array", - json::type_error&); - CHECK_THROWS_WITH_AS(j_string.get(), - "[json.exception.type_error.302] type must be binary, but is string", - json::type_error&); - CHECK_THROWS_WITH_AS(j_boolean.get(), - "[json.exception.type_error.302] type must be binary, but is boolean", - json::type_error&); - - CHECK_THROWS_WITH_AS(j_null_const.get(), - "[json.exception.type_error.302] type must be binary, but is null", - json::type_error&); - CHECK_THROWS_WITH_AS(j_object_const.get(), - "[json.exception.type_error.302] type must be binary, but is object", - json::type_error&); - CHECK_THROWS_WITH_AS(j_array_const.get(), - "[json.exception.type_error.302] type must be binary, but is array", - json::type_error&); - CHECK_THROWS_WITH_AS(j_string_const.get(), - "[json.exception.type_error.302] type must be binary, but is string", - json::type_error&); - CHECK_THROWS_WITH_AS(j_boolean_const.get(), - "[json.exception.type_error.302] type must be binary, but is boolean", - json::type_error&); - - CHECK_THROWS_WITH_AS(j_null.get_binary(), - "[json.exception.type_error.302] type must be binary, but is null", - json::type_error&); - CHECK_THROWS_WITH_AS(j_object.get_binary(), - "[json.exception.type_error.302] type must be binary, but is object", - json::type_error&); - CHECK_THROWS_WITH_AS(j_array.get_binary(), - "[json.exception.type_error.302] type must be binary, but is array", - json::type_error&); - CHECK_THROWS_WITH_AS(j_string.get_binary(), - "[json.exception.type_error.302] type must be binary, but is string", - json::type_error&); - CHECK_THROWS_WITH_AS(j_boolean.get_binary(), - "[json.exception.type_error.302] type must be binary, but is boolean", - json::type_error&); - - CHECK_THROWS_WITH_AS(j_null_const.get_binary(), - "[json.exception.type_error.302] type must be binary, but is null", - json::type_error&); - CHECK_THROWS_WITH_AS(j_object_const.get_binary(), - "[json.exception.type_error.302] type must be binary, but is object", - json::type_error&); - CHECK_THROWS_WITH_AS(j_array_const.get_binary(), - "[json.exception.type_error.302] type must be binary, but is array", - json::type_error&); - CHECK_THROWS_WITH_AS(j_string_const.get_binary(), - "[json.exception.type_error.302] type must be binary, but is string", - json::type_error&); - CHECK_THROWS_WITH_AS(j_boolean_const.get_binary(), - "[json.exception.type_error.302] type must be binary, but is boolean", - json::type_error&); - } - } - -#if JSON_USE_IMPLICIT_CONVERSIONS - SECTION("get a binary value (implicit)") - { - json::binary_t const n_reference{{1, 2, 3}}; - json const j(n_reference); - - SECTION("binary_t") - { - json::binary_t const b = j; - CHECK(*json(b).m_data.m_value.binary == *j.m_data.m_value.binary); - } - } -#endif - -#ifndef SKIP_TESTS_FOR_ENUM_SERIALIZATION - SECTION("get an enum") - { - enum c_enum { value_1, value_2 }; // NOLINT(cppcoreguidelines-use-enum-class) - enum class cpp_enum { value_1, value_2 }; - - CHECK(json(value_1).get() == value_1); - CHECK(json(cpp_enum::value_1).get() == cpp_enum::value_1); - } - - SECTION("get an enum with underlying type bool (#5671)") - { - enum class bool_enum : bool { off, on }; - - CHECK(json(bool_enum::off).get() == bool_enum::off); - CHECK(json(bool_enum::on).get() == bool_enum::on); - } -#endif - - SECTION("more involved conversions") - { - SECTION("object-like STL containers") - { - json const j1 = {{"one", 1}, {"two", 2}, {"three", 3}}; - json const j2 = {{"one", 1u}, {"two", 2u}, {"three", 3u}}; - json const j3 = {{"one", 1.1}, {"two", 2.2}, {"three", 3.3}}; - json const j4 = {{"one", true}, {"two", false}, {"three", true}}; - json const j5 = {{"one", "eins"}, {"two", "zwei"}, {"three", "drei"}}; - - SECTION("std::map") - { - CHECK(j1.get>() == (std::map {{"one", 1}, {"two", 2}, {"three", 3}})); - CHECK(j2.get>() == (std::map {{"one", 1u}, {"two", 2u}, {"three", 3u}})); - CHECK(j3.get>() == (std::map {{"one", 1.1}, {"two", 2.2}, {"three", 3.3}})); - CHECK(j4.get>() == (std::map {{"one", true}, {"two", false}, {"three", true}})); - CHECK(j5.get>() == (std::map {{"one", "eins"}, {"two", "zwei"}, {"three", "drei"}})); - } - - SECTION("std::unordered_map") - { - CHECK(j1.get>() == (std::unordered_map {{"one", 1}, {"two", 2}, {"three", 3}})); - CHECK(j2.get>() == (std::unordered_map {{"one", 1u}, {"two", 2u}, {"three", 3u}})); - CHECK(j3.get>() == (std::unordered_map {{"one", 1.1}, {"two", 2.2}, {"three", 3.3}})); - CHECK(j4.get>() == (std::unordered_map {{"one", true}, {"two", false}, {"three", true}})); - const auto m5 = j5.get>(); - CHECK(m5 == (std::unordered_map {{"one", "eins"}, {"two", "zwei"}, {"three", "drei"}})); - CHECK(m5.at("one") == "eins"); - } - - SECTION("reserve is called on containers that support it (#5406)") - { - // build a larger object so that a missing/incorrect reserve() - // call would be more likely to corrupt or drop elements - json j_large; - for (int i = 0; i < 100; ++i) - { - j_large[std::to_string(i)] = i; - } - - SECTION("std::unordered_map (supports reserve)") - { - const auto m = j_large.get>(); - CHECK(m.size() == 100); - for (int i = 0; i < 100; ++i) - { - CHECK(m.at(std::to_string(i)) == i); - } - } - - SECTION("std::map (no reserve, fallback path)") - { - const auto m = j_large.get>(); - CHECK(m.size() == 100); - for (int i = 0; i < 100; ++i) - { - CHECK(m.at(std::to_string(i)) == i); - } - } - } - - SECTION("std::multimap") - { - CHECK(j1.get>() == (std::multimap {{"one", 1}, {"two", 2}, {"three", 3}})); - CHECK(j2.get>() == (std::multimap {{"one", 1u}, {"two", 2u}, {"three", 3u}})); - CHECK(j3.get>() == (std::multimap {{"one", 1.1}, {"two", 2.2}, {"three", 3.3}})); - CHECK(j4.get>() == (std::multimap {{"one", true}, {"two", false}, {"three", true}})); - const auto m5 = j5.get>(); - CHECK(m5 == (std::multimap {{"one", "eins"}, {"two", "zwei"}, {"three", "drei"}})); - CHECK(m5.find("one")->second == "eins"); - } - - SECTION("std::unordered_multimap") - { - CHECK(j1.get>() == (std::unordered_multimap {{"one", 1}, {"two", 2}, {"three", 3}})); - CHECK(j2.get>() == (std::unordered_multimap {{"one", 1u}, {"two", 2u}, {"three", 3u}})); - CHECK(j3.get>() == (std::unordered_multimap {{"one", 1.1}, {"two", 2.2}, {"three", 3.3}})); - CHECK(j4.get>() == (std::unordered_multimap {{"one", true}, {"two", false}, {"three", true}})); - const auto m5 = j5.get>(); - CHECK(m5 == (std::unordered_multimap {{"one", "eins"}, {"two", "zwei"}, {"three", "drei"}})); - CHECK(m5.find("one")->second == "eins"); - } - - SECTION("exception in case of a non-object type") - { - CHECK_THROWS_WITH_AS( - (json().get>()), - "[json.exception.type_error.302] type must be object, but is null", json::type_error&); - } - } - - SECTION("array-like STL containers") - { - json const j1 = {1, 2, 3, 4}; - json const j2 = {1u, 2u, 3u, 4u}; - json const j3 = {1.2, 2.3, 3.4, 4.5}; - json const j4 = {true, false, true}; - json const j5 = {"one", "two", "three"}; - - SECTION("std::list") - { - CHECK(j1.get>() == (std::list {1, 2, 3, 4})); - CHECK(j2.get>() == (std::list {1u, 2u, 3u, 4u})); - CHECK(j3.get>() == (std::list {1.2, 2.3, 3.4, 4.5})); - CHECK(j4.get>() == (std::list {true, false, true})); - CHECK(j5.get>() == (std::list {"one", "two", "three"})); - } - - SECTION("std::forward_list") - { - CHECK(j1.get>() == (std::forward_list {1, 2, 3, 4})); - CHECK(j2.get>() == (std::forward_list {1u, 2u, 3u, 4u})); - CHECK(j3.get>() == (std::forward_list {1.2, 2.3, 3.4, 4.5})); - CHECK(j4.get>() == (std::forward_list {true, false, true})); - CHECK(j5.get>() == (std::forward_list {"one", "two", "three"})); - } - - SECTION("std::array") - { - CHECK(j1.get>() == (std::array {{1, 2, 3, 4}})); - // only the first 3 elements of j2 are converted, since the target array is smaller - CHECK(j2.get>() == (std::array {{1u, 2u, 3u}})); - CHECK(j3.get>() == (std::array {{1.2, 2.3, 3.4, 4.5}})); - CHECK(j4.get>() == (std::array {{true, false, true}})); - CHECK(j5.get>() == (std::array {{"one", "two", "three"}})); - - SECTION("std::array is larger than JSON") - { - std::array arr6 = {{1, 2, 3, 4, 5, 6}}; - CHECK_THROWS_WITH_AS(j1.get_to(arr6), "[json.exception.out_of_range.401] " - "array index 4 is out of range", json::out_of_range&); - } - - SECTION("std::array is smaller than JSON") - { - std::array arr2 = {{8, 9}}; - j1.get_to(arr2); - CHECK(arr2[0] == 1); - CHECK(arr2[1] == 2); - } - } - - SECTION("std::valarray") - { - // valarray has no operator== that returns bool, so compare via a vector copy - const auto v1 = j1.get>(); - CHECK((std::vector(std::begin(v1), std::end(v1)) == std::vector {1, 2, 3, 4})); - const auto v2 = j2.get>(); - CHECK((std::vector(std::begin(v2), std::end(v2)) == std::vector {1u, 2u, 3u, 4u})); - const auto v3 = j3.get>(); - CHECK((std::vector(std::begin(v3), std::end(v3)) == std::vector {1.2, 2.3, 3.4, 4.5})); - const auto v4 = j4.get>(); - CHECK((std::vector(std::begin(v4), std::end(v4)) == std::vector {true, false, true})); - const auto v5 = j5.get>(); - CHECK((std::vector(std::begin(v5), std::end(v5)) == std::vector {"one", "two", "three"})); - } - - SECTION("std::vector") - { - CHECK(j1.get>() == (std::vector {1, 2, 3, 4})); - CHECK(j2.get>() == (std::vector {1u, 2u, 3u, 4u})); - CHECK(j3.get>() == (std::vector {1.2, 2.3, 3.4, 4.5})); - CHECK(j4.get>() == (std::vector {true, false, true})); - CHECK(j5.get>() == (std::vector {"one", "two", "three"})); - } - - SECTION("std::deque") - { - CHECK(j1.get>() == (std::deque {1, 2, 3, 4})); - CHECK(j2.get>() == (std::deque {1u, 2u, 3u, 4u})); - CHECK(j3.get>() == (std::deque {1.2, 2.3, 3.4, 4.5})); - CHECK(j4.get>() == (std::deque {true, false, true})); - CHECK(j5.get>() == (std::deque {"one", "two", "three"})); - } - - SECTION("std::set") - { - CHECK(j1.get>() == (std::set {1, 2, 3, 4})); - CHECK(j2.get>() == (std::set {1u, 2u, 3u, 4u})); - CHECK(j3.get>() == (std::set {1.2, 2.3, 3.4, 4.5})); - CHECK(j4.get>() == (std::set {true, false, true})); - CHECK(j5.get>() == (std::set {"one", "two", "three"})); - } - - SECTION("std::unordered_set") - { - CHECK(j1.get>() == (std::unordered_set {1, 2, 3, 4})); - CHECK(j2.get>() == (std::unordered_set {1u, 2u, 3u, 4u})); - CHECK(j3.get>() == (std::unordered_set {1.2, 2.3, 3.4, 4.5})); - CHECK(j4.get>() == (std::unordered_set {true, false, true})); - CHECK(j5.get>() == (std::unordered_set {"one", "two", "three"})); - } - - SECTION("std::map (array of pairs)") - { - const std::map m{{0, 1}, {1, 2}, {2, 3}}; - json const j6 = m; - - auto m2 = j6.get>(); - CHECK(m == m2); - - json const j7 = {0, 1, 2, 3}; - json const j8 = 2; -#if JSON_DIAGNOSTICS - CHECK_THROWS_WITH_AS((j7.get>()), - "[json.exception.type_error.302] (/0) type must be array, " - "but is number", json::type_error&); -#else - CHECK_THROWS_WITH_AS((j7.get>()), - "[json.exception.type_error.302] type must be array, " - "but is number", json::type_error&); -#endif - CHECK_THROWS_WITH_AS((j8.get>()), - "[json.exception.type_error.302] type must be array, " - "but is number", json::type_error&); - - SECTION("superfluous entries") - { - json const j9 = {{0, 1, 2}, {1, 2, 3}, {2, 3, 4}}; - m2 = j9.get>(); - CHECK(m == m2); - } - } - - SECTION("std::unordered_map (array of pairs)") - { - const std::unordered_map m{{0, 1}, {1, 2}, {2, 3}}; - json const j6 = m; - - auto m2 = j6.get>(); - CHECK(m == m2); - - json const j7 = {0, 1, 2, 3}; - json const j8 = 2; -#if JSON_DIAGNOSTICS - CHECK_THROWS_WITH_AS((j7.get>()), - "[json.exception.type_error.302] (/0) type must be array, " - "but is number", json::type_error&); -#else - CHECK_THROWS_WITH_AS((j7.get>()), - "[json.exception.type_error.302] type must be array, " - "but is number", json::type_error&); -#endif - CHECK_THROWS_WITH_AS((j8.get>()), - "[json.exception.type_error.302] type must be array, " - "but is number", json::type_error&); - - SECTION("superfluous entries") - { - json const j9{{0, 1, 2}, {1, 2, 3}, {2, 3, 4}}; - m2 = j9.get>(); - CHECK(m == m2); - } - } - - SECTION("exception in case of a non-object type") - { - // does type really must be an array? or it rather must not be null? - // that's what I thought when other test like this one broke - CHECK_THROWS_WITH_AS( - (json().get>()), - "[json.exception.type_error.302] type must be array, but is null", json::type_error&); - CHECK_THROWS_WITH_AS( - (json().get>()), - "[json.exception.type_error.302] type must be array, but is null", json::type_error&); - CHECK_THROWS_WITH_AS( - (json().get>()), - "[json.exception.type_error.302] type must be array, but is null", json::type_error&); - CHECK_THROWS_WITH_AS( - (json().get>()), - "[json.exception.type_error.302] type must be array, but is null", json::type_error&); - CHECK_THROWS_WITH_AS( - (json().get>()), - "[json.exception.type_error.302] type must be array, but is null", json::type_error&); - CHECK_THROWS_WITH_AS( - (json().get>()), - "[json.exception.type_error.302] type must be array, but is null", json::type_error&); - } - } - } } -enum class cards {kreuz, pik, herz, karo}; - -// NOLINTNEXTLINE(misc-use-internal-linkage,misc-const-correctness) - false positive -NLOHMANN_JSON_SERIALIZE_ENUM(cards, -{ - {cards::kreuz, "kreuz"}, - {cards::pik, "pik"}, - {cards::pik, "puk"}, // second entry for cards::puk; will not be used - {cards::herz, "herz"}, - {cards::karo, "karo"} -}) - -enum TaskState // NOLINT(cert-int09-c,readability-enum-initial-value,cppcoreguidelines-use-enum-class) -{ - TS_STOPPED, - TS_RUNNING, - TS_COMPLETED, - TS_INVALID = -1, -}; - -// NOLINTNEXTLINE(misc-const-correctness,misc-use-internal-linkage) - false positive -NLOHMANN_JSON_SERIALIZE_ENUM(TaskState, -{ - {TS_INVALID, nullptr}, - {TS_STOPPED, "stopped"}, - {TS_RUNNING, "running"}, - {TS_COMPLETED, "completed"}, -}) - -TEST_CASE("JSON to enum mapping") -{ - SECTION("enum class") - { - // enum -> json - CHECK(json(cards::kreuz) == "kreuz"); - CHECK(json(cards::pik) == "pik"); - CHECK(json(cards::herz) == "herz"); - CHECK(json(cards::karo) == "karo"); - - // json -> enum - CHECK(cards::kreuz == json("kreuz")); - CHECK(cards::pik == json("pik")); - CHECK(cards::herz == json("herz")); - CHECK(cards::karo == json("karo")); - - // invalid json -> first enum - CHECK(cards::kreuz == json("what?").get()); - } - - SECTION("traditional enum") - { - // enum -> json - CHECK(json(TS_STOPPED) == "stopped"); - CHECK(json(TS_RUNNING) == "running"); - CHECK(json(TS_COMPLETED) == "completed"); - CHECK(json(TS_INVALID) == json()); - - // json -> enum - CHECK(TS_STOPPED == json("stopped")); - CHECK(TS_RUNNING == json("running")); - CHECK(TS_COMPLETED == json("completed")); - CHECK(TS_INVALID == json()); - - // invalid json -> first enum - CHECK(TS_INVALID == json("what?").get()); - } -} - -enum class strict_cards {kreuz, pik, herz, karo, andere}; // andere not included in mapping - -// NOLINTNEXTLINE(misc-use-internal-linkage,misc-const-correctness) - false positive -NLOHMANN_JSON_SERIALIZE_ENUM_STRICT(strict_cards, -{ - {strict_cards::kreuz, "kreuz"}, - {strict_cards::pik, "pik"}, - {strict_cards::pik, "puk"}, // second entry for cards::pik; will not be used - {strict_cards::herz, "herz"}, - {strict_cards::karo, "karo"} -}) - -enum StrictTaskState // NOLINT(cert-int09-c,readability-enum-initial-value,cppcoreguidelines-use-enum-class) -{ - STRICT_TS_STOPPED, - STRICT_TS_RUNNING, - STRICT_TS_COMPLETED, - STRICT_TS_OTHER, // STRICT_TS_OTHER not in mapping - STRICT_TS_INVALID = -1, -}; - -// NOLINTNEXTLINE(misc-const-correctness,misc-use-internal-linkage) - false positive -NLOHMANN_JSON_SERIALIZE_ENUM_STRICT(StrictTaskState, -{ - {STRICT_TS_INVALID, nullptr}, - {STRICT_TS_STOPPED, "stopped"}, - {STRICT_TS_RUNNING, "running"}, - {STRICT_TS_COMPLETED, "completed"}, -}) - -// regression test for #5708 item 2: NLOHMANN_JSON_SERIALIZE_ENUM_STRICT must not rely on -// unqualified lookup of a helper name that a user's own namespace may also declare -namespace ns_with_colliding_name -{ -// NOLINTNEXTLINE(misc-use-internal-linkage) - used to shadow the library's internal helper name -inline void templated_json_throw(int /*unused*/) {} - -enum class colliding_enum { a, b }; - -// NOLINTNEXTLINE(misc-use-internal-linkage,misc-const-correctness,cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) - false positive -NLOHMANN_JSON_SERIALIZE_ENUM_STRICT(colliding_enum, -{ - {colliding_enum::a, "a"}, - {colliding_enum::b, "b"} -}) -} // namespace ns_with_colliding_name - -TEST_CASE("NLOHMANN_JSON_SERIALIZE_ENUM_STRICT in a namespace with a colliding name") -{ - using ns_with_colliding_name::colliding_enum; - - CHECK(json(colliding_enum::a) == "a"); - CHECK(colliding_enum::b == json("b")); - - json _; - CHECK_THROWS_WITH_AS(_ = json("nope").get(), "[json.exception.out_of_range.410] enum value out of range for colliding_enum: \"nope\"", json::out_of_range&); -} - -TEST_CASE("Strict JSON to enum mapping") -{ - SECTION("enum class") - { - // enum -> json - CHECK(json(strict_cards::kreuz) == "kreuz"); - CHECK(json(strict_cards::pik) == "pik"); - CHECK(json(strict_cards::herz) == "herz"); - CHECK(json(strict_cards::karo) == "karo"); - - // json -> enum - CHECK(json("kreuz").get() == strict_cards::kreuz); - CHECK(json("pik").get() == strict_cards::pik); - CHECK(json("herz").get() == strict_cards::herz); - CHECK(json("karo").get() == strict_cards::karo); - - // comparison of enum and json - CHECK(strict_cards::kreuz == json("kreuz")); - CHECK(strict_cards::pik == json("pik")); - CHECK(strict_cards::herz == json("herz")); - CHECK(strict_cards::karo == json("karo")); - - // invalid json -> exception thrown - json _; - CHECK_THROWS_WITH_AS(_ = json("what?").get(), "[json.exception.out_of_range.410] enum value out of range for strict_cards: \"what?\"", json::out_of_range&); - - // conversion of unmapped enum -> exception thrown - CHECK_THROWS_WITH_AS(json(strict_cards::andere), "[json.exception.out_of_range.410] enum value out of range for strict_cards", json::out_of_range&); - - // comparing an unmapped enum with json throws the same exception - // (the scalar comparison operators used to be noexcept, so this - // called std::terminate) - CHECK_THROWS_WITH_AS(static_cast(strict_cards::andere == json("andere")), "[json.exception.out_of_range.410] enum value out of range for strict_cards", json::out_of_range&); - CHECK_THROWS_WITH_AS(static_cast(json("andere") != strict_cards::andere), "[json.exception.out_of_range.410] enum value out of range for strict_cards", json::out_of_range&); - - // invalid UTF-8 -> out_of_range.410, not the type_error.316 thrown while building the - // message (regression test for #5667); such strings can reach get() unvalidated, - // e.g. from from_cbor()/from_msgpack() (#5529) - const json j_invalid_utf8 = "\xFF"; - CHECK_THROWS_WITH_AS(_ = j_invalid_utf8.get(), "[json.exception.out_of_range.410] enum value out of range for strict_cards: \"\xEF\xBF\xBD\"", json::out_of_range&); - } - - SECTION("traditional enum") - { - // enum -> json - CHECK(json(STRICT_TS_STOPPED) == "stopped"); - CHECK(json(STRICT_TS_RUNNING) == "running"); - CHECK(json(STRICT_TS_COMPLETED) == "completed"); - CHECK(json(STRICT_TS_INVALID) == json()); - - // json -> enum - CHECK(json("stopped").get() == STRICT_TS_STOPPED); - CHECK(json("running").get() == STRICT_TS_RUNNING); - CHECK(json("completed").get() == STRICT_TS_COMPLETED); - CHECK(json().get() == STRICT_TS_INVALID); - - // comparison of enum and json - CHECK(STRICT_TS_STOPPED == json("stopped")); - CHECK(STRICT_TS_RUNNING == json("running")); - CHECK(STRICT_TS_COMPLETED == json("completed")); - CHECK(STRICT_TS_INVALID == json()); - - // invalid json -> exception thrown - json _; - CHECK_THROWS_WITH_AS(_ = json("what?").get(), "[json.exception.out_of_range.410] enum value out of range for StrictTaskState: \"what?\"", json::out_of_range&); - - // conversion of unmapped enum -> exception thrown - CHECK_THROWS_WITH_AS(json(STRICT_TS_OTHER), "[json.exception.out_of_range.410] enum value out of range for StrictTaskState", json::out_of_range&); - - // comparing an unmapped enum with json throws the same exception - CHECK_THROWS_WITH_AS(static_cast(STRICT_TS_OTHER < json("x")), "[json.exception.out_of_range.410] enum value out of range for StrictTaskState", json::out_of_range&); - } -} - - -#ifdef JSON_HAS_CPP_17 -#if JSON_HAS_FILESYSTEM || JSON_HAS_EXPERIMENTAL_FILESYSTEM -TEST_CASE("std::filesystem::path") -{ - SECTION("ascii") - { - json const j_string = "Path"; - auto p = j_string.template get(); - json const j_path = p; - - CHECK(j_path.template get() == - j_string.template get()); - } - - SECTION("utf-8") - { - json const j_string = "P\xc4\x9b\xc5\xa1ina"; - auto p = j_string.template get(); - json const j_path = p; - - CHECK(j_path.template get() == - j_string.template get()); - } -} -#endif - -// the ADL to_json overload for std::u8string only exists under the same guard -// as std::filesystem::path support (it is otherwise only reached indirectly, -// via std::filesystem::path::u8string()) -- mirror both #if conditions from -// include/nlohmann/detail/conversions/to_json.hpp exactly -#if JSON_HAS_FILESYSTEM || JSON_HAS_EXPERIMENTAL_FILESYSTEM -#if defined(__cpp_lib_char8_t) -TEST_CASE("std::u8string") -{ - SECTION("ascii") - { - const std::u8string s = u8"Path"; - json const j = s; - - CHECK(j.template get() == "Path"); - } - - SECTION("utf-8") - { - // use \u universal-character-names (rather than raw \x byte escapes - // or literal non-ASCII source bytes) to compose the multi-byte UTF-8 - // encoding -- MSVC treats \x escapes used that way inside a u8 - // literal as a nonstandard extension (warning C5321), which some of - // our CI configs promote to an error; \u is portable and produces - // the exact same encoded bytes without depending on the source - // file's encoding - const std::u8string s = u8"P\u011B\u0161ina"; - json const j = s; - - CHECK(j.template get() == "P\xc4\x9b\xc5\xa1ina"); - } -} -#endif -#endif - -#if !defined(JSON_NOEXCEPTION) -namespace -{ -// a type whose to_json reports an error by throwing, used below to check that -// converting a std::optional to JSON propagates an exception thrown while -// converting its contained value instead of calling std::terminate (#5642) -struct throwing_to_json_type {}; - -[[noreturn]] void to_json(json& /*unused*/, const throwing_to_json_type& /*unused*/) -{ - throw std::runtime_error("cannot serialize throwing_to_json_type"); -} -} // namespace -#endif - -TEST_CASE("std::optional") -{ - SECTION("null") - { - const json j_null; - const std::optional opt_null; - - CHECK(json(opt_null) == j_null); - CHECK(j_null.get>() == std::nullopt); - - // Constructing std::optional directly from JSON null throws because - // std::optional's own converting constructor is chosen over basic_json's - // operator T(). This is a language-level limitation (std::optional is - // constructible from T, and T is constructible from basic_json via the - // operator); there is no SFINAE path that distinguishes "call from inside - // std::optional's constructor" from "direct call". Use get>() - // or get_to() instead for correct null handling. See #4864 and #5246. - CHECK_THROWS_WITH_AS(std::optional(j_null), - "[json.exception.type_error.302] type must be string, but is null", json::type_error&); - CHECK_THROWS_WITH_AS(std::optional(j_null), - "[json.exception.type_error.302] type must be number, but is null", json::type_error&); - - // Assignment goes through the same overload resolution as direct - // construction, so it throws for the same reason. This relies on - // basic_json's implicit conversion operator, so it only applies - // when JSON_USE_IMPLICIT_CONVERSIONS is enabled (the default). -#if JSON_USE_IMPLICIT_CONVERSIONS - std::optional opt_assign; - CHECK_THROWS_WITH_AS(opt_assign = j_null, - "[json.exception.type_error.302] type must be string, but is null", json::type_error&); -#endif - - // get_to() is the correct way to obtain std::nullopt from a JSON null. - std::optional opt_get_to = "placeholder"; - j_null.get_to(opt_get_to); - CHECK(opt_get_to == std::nullopt); - } - - SECTION("string") - { - json j_string = "string"; - std::optional opt_string = "string"; - - CHECK(json(opt_string) == j_string); - CHECK(std::optional(j_string) == opt_string); - // false positive: Infer attributes the destruction of the temporaries above to opt_string - // @infer-ignore USE_AFTER_DELETE - } - - SECTION("bool") - { - json j_bool = true; - std::optional opt_bool = true; - - CHECK(json(opt_bool) == j_bool); - CHECK(std::optional(j_bool) == opt_bool); - } - - SECTION("number") - { - json j_number = 1; - std::optional opt_int = 1; - - CHECK(json(opt_int) == j_number); - CHECK(j_number.get>() == opt_int); - } - - SECTION("array") - { - json j_array = {1, 2, nullptr}; - std::vector> opt_array = {{1, 2, std::nullopt}}; - - CHECK(json(opt_array) == j_array); - CHECK(j_array.get>>() == opt_array); - } - - SECTION("object") - { - json j_object = {{"one", 1}, {"two", 2}, {"zero", nullptr}}; - std::map> opt_object {{"one", 1}, {"two", 2}, {"zero", std::nullopt}}; - - CHECK(json(opt_object) == j_object); - CHECK(std::map>(j_object) == opt_object); - } - -#if !defined(JSON_NOEXCEPTION) - SECTION("exception from contained value's to_json propagates (#5642)") - { - // to_json(BasicJsonType&, const std::optional&) must not be - // noexcept: it calls T's to_json, which may throw (a user-defined - // to_json that reports an error, or std::bad_alloc for T = - // std::string/vector/json). Before the fix, this called - // std::terminate() instead of letting the exception propagate. - const std::optional opt = throwing_to_json_type{}; - CHECK_THROWS_WITH_AS(json(opt), "cannot serialize throwing_to_json_type", std::runtime_error&); - - // the conversion is noexcept exactly when converting the contained value is - static_assert(!std::is_nothrow_constructible&>::value); - static_assert(std::is_nothrow_constructible&>::value); - } -#endif -} -#endif - #ifdef JSON_HAS_CPP_17 #undef JSON_HAS_CPP_17 #endif diff --git a/tests/src/unit-conversions2.cpp b/tests/src/unit-conversions2.cpp new file mode 100644 index 000000000..eca482b83 --- /dev/null +++ b/tests/src/unit-conversions2.cpp @@ -0,0 +1,868 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +// cmake/test.cmake selects the C++ standard versions with which to build a +// unit test based on the presence of JSON_HAS_CPP_ macros. +// When using macros that are only defined for particular versions of the standard +// (e.g., JSON_HAS_FILESYSTEM for C++17 and up), please mention the corresponding +// version macro in a comment close by, like this: +// JSON_HAS_CPP_ (do not remove; see note at top of file) + +#include "doctest_compatibility.h" + +// skip tests if JSON_DisableEnumSerialization=ON (#4384) +#if defined(JSON_DISABLE_ENUM_SERIALIZATION) && (JSON_DISABLE_ENUM_SERIALIZATION == 1) + #define SKIP_TESTS_FOR_ENUM_SERIALIZATION +#endif + +#define JSON_TESTS_PRIVATE +#include +using nlohmann::json; + +#include +#include +#include +#include +#include +#include +#include + +// NLOHMANN_JSON_SERIALIZE_ENUM uses a static std::pair +DOCTEST_CLANG_SUPPRESS_WARNING_PUSH +DOCTEST_CLANG_SUPPRESS_WARNING("-Wexit-time-destructors") + +#if (defined(__cplusplus) && __cplusplus >= 201703L) || (defined(_HAS_CXX17) && _HAS_CXX17 == 1) // fix for issue #464 + #define JSON_HAS_CPP_17 + #define JSON_HAS_CPP_14 +#elif (defined(__cplusplus) && __cplusplus >= 201402L) || (defined(_HAS_CXX14) && _HAS_CXX14 == 1) + #define JSON_HAS_CPP_14 +#endif + +#ifdef JSON_HAS_CPP_17 + #if __has_include() + #include + #elif __has_include() + #include + #endif +#endif + +#if defined(JSON_HAS_CPP_17) + #include +#endif + +TEST_CASE("value conversion") +{ + SECTION("get a binary value (explicit)") + { + json::binary_t const n_reference{{1, 2, 3}}; + json j(n_reference); + + SECTION("binary_t") + { + json::binary_t const b = j.get(); + CHECK(*json(b).m_data.m_value.binary == *j.m_data.m_value.binary); + } + + SECTION("get_binary()") + { + SECTION("non-const") + { + auto& b = j.get_binary(); + CHECK(*json(b).m_data.m_value.binary == *j.m_data.m_value.binary); + } + + SECTION("non-const") + { + const json j_const = j; // NOLINT(performance-unnecessary-copy-initialization) + const auto& b = j_const.get_binary(); + CHECK(*json(b).m_data.m_value.binary == *j.m_data.m_value.binary); + } + } + + SECTION("exception in case of a non-string type") + { + json j_null(json::value_t::null); + json j_object(json::value_t::object); + json j_array(json::value_t::array); + json j_string(json::value_t::string); + json j_boolean(json::value_t::boolean); + const json j_null_const(json::value_t::null); + const json j_object_const(json::value_t::object); + const json j_array_const(json::value_t::array); + const json j_string_const(json::value_t::string); + const json j_boolean_const(json::value_t::boolean); + + CHECK_THROWS_WITH_AS(j_null.get(), + "[json.exception.type_error.302] type must be binary, but is null", + json::type_error&); + CHECK_THROWS_WITH_AS(j_object.get(), + "[json.exception.type_error.302] type must be binary, but is object", + json::type_error&); + CHECK_THROWS_WITH_AS(j_array.get(), + "[json.exception.type_error.302] type must be binary, but is array", + json::type_error&); + CHECK_THROWS_WITH_AS(j_string.get(), + "[json.exception.type_error.302] type must be binary, but is string", + json::type_error&); + CHECK_THROWS_WITH_AS(j_boolean.get(), + "[json.exception.type_error.302] type must be binary, but is boolean", + json::type_error&); + + CHECK_THROWS_WITH_AS(j_null_const.get(), + "[json.exception.type_error.302] type must be binary, but is null", + json::type_error&); + CHECK_THROWS_WITH_AS(j_object_const.get(), + "[json.exception.type_error.302] type must be binary, but is object", + json::type_error&); + CHECK_THROWS_WITH_AS(j_array_const.get(), + "[json.exception.type_error.302] type must be binary, but is array", + json::type_error&); + CHECK_THROWS_WITH_AS(j_string_const.get(), + "[json.exception.type_error.302] type must be binary, but is string", + json::type_error&); + CHECK_THROWS_WITH_AS(j_boolean_const.get(), + "[json.exception.type_error.302] type must be binary, but is boolean", + json::type_error&); + + CHECK_THROWS_WITH_AS(j_null.get_binary(), + "[json.exception.type_error.302] type must be binary, but is null", + json::type_error&); + CHECK_THROWS_WITH_AS(j_object.get_binary(), + "[json.exception.type_error.302] type must be binary, but is object", + json::type_error&); + CHECK_THROWS_WITH_AS(j_array.get_binary(), + "[json.exception.type_error.302] type must be binary, but is array", + json::type_error&); + CHECK_THROWS_WITH_AS(j_string.get_binary(), + "[json.exception.type_error.302] type must be binary, but is string", + json::type_error&); + CHECK_THROWS_WITH_AS(j_boolean.get_binary(), + "[json.exception.type_error.302] type must be binary, but is boolean", + json::type_error&); + + CHECK_THROWS_WITH_AS(j_null_const.get_binary(), + "[json.exception.type_error.302] type must be binary, but is null", + json::type_error&); + CHECK_THROWS_WITH_AS(j_object_const.get_binary(), + "[json.exception.type_error.302] type must be binary, but is object", + json::type_error&); + CHECK_THROWS_WITH_AS(j_array_const.get_binary(), + "[json.exception.type_error.302] type must be binary, but is array", + json::type_error&); + CHECK_THROWS_WITH_AS(j_string_const.get_binary(), + "[json.exception.type_error.302] type must be binary, but is string", + json::type_error&); + CHECK_THROWS_WITH_AS(j_boolean_const.get_binary(), + "[json.exception.type_error.302] type must be binary, but is boolean", + json::type_error&); + } + } + +#if JSON_USE_IMPLICIT_CONVERSIONS + SECTION("get a binary value (implicit)") + { + json::binary_t const n_reference{{1, 2, 3}}; + json const j(n_reference); + + SECTION("binary_t") + { + json::binary_t const b = j; + CHECK(*json(b).m_data.m_value.binary == *j.m_data.m_value.binary); + } + } +#endif + +#ifndef SKIP_TESTS_FOR_ENUM_SERIALIZATION + SECTION("get an enum") + { + enum c_enum { value_1, value_2 }; // NOLINT(cppcoreguidelines-use-enum-class) + enum class cpp_enum { value_1, value_2 }; + + CHECK(json(value_1).get() == value_1); + CHECK(json(cpp_enum::value_1).get() == cpp_enum::value_1); + } + + SECTION("get an enum with underlying type bool (#5671)") + { + enum class bool_enum : bool { off, on }; + + CHECK(json(bool_enum::off).get() == bool_enum::off); + CHECK(json(bool_enum::on).get() == bool_enum::on); + } +#endif + + SECTION("more involved conversions") + { + SECTION("object-like STL containers") + { + json const j1 = {{"one", 1}, {"two", 2}, {"three", 3}}; + json const j2 = {{"one", 1u}, {"two", 2u}, {"three", 3u}}; + json const j3 = {{"one", 1.1}, {"two", 2.2}, {"three", 3.3}}; + json const j4 = {{"one", true}, {"two", false}, {"three", true}}; + json const j5 = {{"one", "eins"}, {"two", "zwei"}, {"three", "drei"}}; + + SECTION("std::map") + { + CHECK(j1.get>() == (std::map {{"one", 1}, {"two", 2}, {"three", 3}})); + CHECK(j2.get>() == (std::map {{"one", 1u}, {"two", 2u}, {"three", 3u}})); + CHECK(j3.get>() == (std::map {{"one", 1.1}, {"two", 2.2}, {"three", 3.3}})); + CHECK(j4.get>() == (std::map {{"one", true}, {"two", false}, {"three", true}})); + CHECK(j5.get>() == (std::map {{"one", "eins"}, {"two", "zwei"}, {"three", "drei"}})); + } + + SECTION("std::unordered_map") + { + CHECK(j1.get>() == (std::unordered_map {{"one", 1}, {"two", 2}, {"three", 3}})); + CHECK(j2.get>() == (std::unordered_map {{"one", 1u}, {"two", 2u}, {"three", 3u}})); + CHECK(j3.get>() == (std::unordered_map {{"one", 1.1}, {"two", 2.2}, {"three", 3.3}})); + CHECK(j4.get>() == (std::unordered_map {{"one", true}, {"two", false}, {"three", true}})); + const auto m5 = j5.get>(); + CHECK(m5 == (std::unordered_map {{"one", "eins"}, {"two", "zwei"}, {"three", "drei"}})); + CHECK(m5.at("one") == "eins"); + } + + SECTION("reserve is called on containers that support it (#5406)") + { + // build a larger object so that a missing/incorrect reserve() + // call would be more likely to corrupt or drop elements + json j_large; + for (int i = 0; i < 100; ++i) + { + j_large[std::to_string(i)] = i; + } + + SECTION("std::unordered_map (supports reserve)") + { + const auto m = j_large.get>(); + CHECK(m.size() == 100); + for (int i = 0; i < 100; ++i) + { + CHECK(m.at(std::to_string(i)) == i); + } + } + + SECTION("std::map (no reserve, fallback path)") + { + const auto m = j_large.get>(); + CHECK(m.size() == 100); + for (int i = 0; i < 100; ++i) + { + CHECK(m.at(std::to_string(i)) == i); + } + } + } + + SECTION("std::multimap") + { + CHECK(j1.get>() == (std::multimap {{"one", 1}, {"two", 2}, {"three", 3}})); + CHECK(j2.get>() == (std::multimap {{"one", 1u}, {"two", 2u}, {"three", 3u}})); + CHECK(j3.get>() == (std::multimap {{"one", 1.1}, {"two", 2.2}, {"three", 3.3}})); + CHECK(j4.get>() == (std::multimap {{"one", true}, {"two", false}, {"three", true}})); + const auto m5 = j5.get>(); + CHECK(m5 == (std::multimap {{"one", "eins"}, {"two", "zwei"}, {"three", "drei"}})); + CHECK(m5.find("one")->second == "eins"); + } + + SECTION("std::unordered_multimap") + { + CHECK(j1.get>() == (std::unordered_multimap {{"one", 1}, {"two", 2}, {"three", 3}})); + CHECK(j2.get>() == (std::unordered_multimap {{"one", 1u}, {"two", 2u}, {"three", 3u}})); + CHECK(j3.get>() == (std::unordered_multimap {{"one", 1.1}, {"two", 2.2}, {"three", 3.3}})); + CHECK(j4.get>() == (std::unordered_multimap {{"one", true}, {"two", false}, {"three", true}})); + const auto m5 = j5.get>(); + CHECK(m5 == (std::unordered_multimap {{"one", "eins"}, {"two", "zwei"}, {"three", "drei"}})); + CHECK(m5.find("one")->second == "eins"); + } + + SECTION("exception in case of a non-object type") + { + CHECK_THROWS_WITH_AS( + (json().get>()), + "[json.exception.type_error.302] type must be object, but is null", json::type_error&); + } + } + + SECTION("array-like STL containers") + { + json const j1 = {1, 2, 3, 4}; + json const j2 = {1u, 2u, 3u, 4u}; + json const j3 = {1.2, 2.3, 3.4, 4.5}; + json const j4 = {true, false, true}; + json const j5 = {"one", "two", "three"}; + + SECTION("std::list") + { + CHECK(j1.get>() == (std::list {1, 2, 3, 4})); + CHECK(j2.get>() == (std::list {1u, 2u, 3u, 4u})); + CHECK(j3.get>() == (std::list {1.2, 2.3, 3.4, 4.5})); + CHECK(j4.get>() == (std::list {true, false, true})); + CHECK(j5.get>() == (std::list {"one", "two", "three"})); + } + + SECTION("std::forward_list") + { + CHECK(j1.get>() == (std::forward_list {1, 2, 3, 4})); + CHECK(j2.get>() == (std::forward_list {1u, 2u, 3u, 4u})); + CHECK(j3.get>() == (std::forward_list {1.2, 2.3, 3.4, 4.5})); + CHECK(j4.get>() == (std::forward_list {true, false, true})); + CHECK(j5.get>() == (std::forward_list {"one", "two", "three"})); + } + + SECTION("std::array") + { + CHECK(j1.get>() == (std::array {{1, 2, 3, 4}})); + // only the first 3 elements of j2 are converted, since the target array is smaller + CHECK(j2.get>() == (std::array {{1u, 2u, 3u}})); + CHECK(j3.get>() == (std::array {{1.2, 2.3, 3.4, 4.5}})); + CHECK(j4.get>() == (std::array {{true, false, true}})); + CHECK(j5.get>() == (std::array {{"one", "two", "three"}})); + + SECTION("std::array is larger than JSON") + { + std::array arr6 = {{1, 2, 3, 4, 5, 6}}; + CHECK_THROWS_WITH_AS(j1.get_to(arr6), "[json.exception.out_of_range.401] " + "array index 4 is out of range", json::out_of_range&); + } + + SECTION("std::array is smaller than JSON") + { + std::array arr2 = {{8, 9}}; + j1.get_to(arr2); + CHECK(arr2[0] == 1); + CHECK(arr2[1] == 2); + } + } + + SECTION("std::valarray") + { + // valarray has no operator== that returns bool, so compare via a vector copy + const auto v1 = j1.get>(); + CHECK((std::vector(std::begin(v1), std::end(v1)) == std::vector {1, 2, 3, 4})); + const auto v2 = j2.get>(); + CHECK((std::vector(std::begin(v2), std::end(v2)) == std::vector {1u, 2u, 3u, 4u})); + const auto v3 = j3.get>(); + CHECK((std::vector(std::begin(v3), std::end(v3)) == std::vector {1.2, 2.3, 3.4, 4.5})); + const auto v4 = j4.get>(); + CHECK((std::vector(std::begin(v4), std::end(v4)) == std::vector {true, false, true})); + const auto v5 = j5.get>(); + CHECK((std::vector(std::begin(v5), std::end(v5)) == std::vector {"one", "two", "three"})); + } + + SECTION("std::vector") + { + CHECK(j1.get>() == (std::vector {1, 2, 3, 4})); + CHECK(j2.get>() == (std::vector {1u, 2u, 3u, 4u})); + CHECK(j3.get>() == (std::vector {1.2, 2.3, 3.4, 4.5})); + CHECK(j4.get>() == (std::vector {true, false, true})); + CHECK(j5.get>() == (std::vector {"one", "two", "three"})); + } + + SECTION("std::deque") + { + CHECK(j1.get>() == (std::deque {1, 2, 3, 4})); + CHECK(j2.get>() == (std::deque {1u, 2u, 3u, 4u})); + CHECK(j3.get>() == (std::deque {1.2, 2.3, 3.4, 4.5})); + CHECK(j4.get>() == (std::deque {true, false, true})); + CHECK(j5.get>() == (std::deque {"one", "two", "three"})); + } + + SECTION("std::set") + { + CHECK(j1.get>() == (std::set {1, 2, 3, 4})); + CHECK(j2.get>() == (std::set {1u, 2u, 3u, 4u})); + CHECK(j3.get>() == (std::set {1.2, 2.3, 3.4, 4.5})); + CHECK(j4.get>() == (std::set {true, false, true})); + CHECK(j5.get>() == (std::set {"one", "two", "three"})); + } + + SECTION("std::unordered_set") + { + CHECK(j1.get>() == (std::unordered_set {1, 2, 3, 4})); + CHECK(j2.get>() == (std::unordered_set {1u, 2u, 3u, 4u})); + CHECK(j3.get>() == (std::unordered_set {1.2, 2.3, 3.4, 4.5})); + CHECK(j4.get>() == (std::unordered_set {true, false, true})); + CHECK(j5.get>() == (std::unordered_set {"one", "two", "three"})); + } + + SECTION("std::map (array of pairs)") + { + const std::map m{{0, 1}, {1, 2}, {2, 3}}; + json const j6 = m; + + auto m2 = j6.get>(); + CHECK(m == m2); + + json const j7 = {0, 1, 2, 3}; + json const j8 = 2; +#if JSON_DIAGNOSTICS + CHECK_THROWS_WITH_AS((j7.get>()), + "[json.exception.type_error.302] (/0) type must be array, " + "but is number", json::type_error&); +#else + CHECK_THROWS_WITH_AS((j7.get>()), + "[json.exception.type_error.302] type must be array, " + "but is number", json::type_error&); +#endif + CHECK_THROWS_WITH_AS((j8.get>()), + "[json.exception.type_error.302] type must be array, " + "but is number", json::type_error&); + + SECTION("superfluous entries") + { + json const j9 = {{0, 1, 2}, {1, 2, 3}, {2, 3, 4}}; + m2 = j9.get>(); + CHECK(m == m2); + } + } + + SECTION("std::unordered_map (array of pairs)") + { + const std::unordered_map m{{0, 1}, {1, 2}, {2, 3}}; + json const j6 = m; + + auto m2 = j6.get>(); + CHECK(m == m2); + + json const j7 = {0, 1, 2, 3}; + json const j8 = 2; +#if JSON_DIAGNOSTICS + CHECK_THROWS_WITH_AS((j7.get>()), + "[json.exception.type_error.302] (/0) type must be array, " + "but is number", json::type_error&); +#else + CHECK_THROWS_WITH_AS((j7.get>()), + "[json.exception.type_error.302] type must be array, " + "but is number", json::type_error&); +#endif + CHECK_THROWS_WITH_AS((j8.get>()), + "[json.exception.type_error.302] type must be array, " + "but is number", json::type_error&); + + SECTION("superfluous entries") + { + json const j9{{0, 1, 2}, {1, 2, 3}, {2, 3, 4}}; + m2 = j9.get>(); + CHECK(m == m2); + } + } + + SECTION("exception in case of a non-object type") + { + // does type really must be an array? or it rather must not be null? + // that's what I thought when other test like this one broke + CHECK_THROWS_WITH_AS( + (json().get>()), + "[json.exception.type_error.302] type must be array, but is null", json::type_error&); + CHECK_THROWS_WITH_AS( + (json().get>()), + "[json.exception.type_error.302] type must be array, but is null", json::type_error&); + CHECK_THROWS_WITH_AS( + (json().get>()), + "[json.exception.type_error.302] type must be array, but is null", json::type_error&); + CHECK_THROWS_WITH_AS( + (json().get>()), + "[json.exception.type_error.302] type must be array, but is null", json::type_error&); + CHECK_THROWS_WITH_AS( + (json().get>()), + "[json.exception.type_error.302] type must be array, but is null", json::type_error&); + CHECK_THROWS_WITH_AS( + (json().get>()), + "[json.exception.type_error.302] type must be array, but is null", json::type_error&); + } + } + } +} + +enum class cards {kreuz, pik, herz, karo}; + +// NOLINTNEXTLINE(misc-use-internal-linkage,misc-const-correctness) - false positive +NLOHMANN_JSON_SERIALIZE_ENUM(cards, +{ + {cards::kreuz, "kreuz"}, + {cards::pik, "pik"}, + {cards::pik, "puk"}, // second entry for cards::puk; will not be used + {cards::herz, "herz"}, + {cards::karo, "karo"} +}) + +enum TaskState // NOLINT(cert-int09-c,readability-enum-initial-value,cppcoreguidelines-use-enum-class) +{ + TS_STOPPED, + TS_RUNNING, + TS_COMPLETED, + TS_INVALID = -1, +}; + +// NOLINTNEXTLINE(misc-const-correctness,misc-use-internal-linkage) - false positive +NLOHMANN_JSON_SERIALIZE_ENUM(TaskState, +{ + {TS_INVALID, nullptr}, + {TS_STOPPED, "stopped"}, + {TS_RUNNING, "running"}, + {TS_COMPLETED, "completed"}, +}) + +TEST_CASE("JSON to enum mapping") +{ + SECTION("enum class") + { + // enum -> json + CHECK(json(cards::kreuz) == "kreuz"); + CHECK(json(cards::pik) == "pik"); + CHECK(json(cards::herz) == "herz"); + CHECK(json(cards::karo) == "karo"); + + // json -> enum + CHECK(cards::kreuz == json("kreuz")); + CHECK(cards::pik == json("pik")); + CHECK(cards::herz == json("herz")); + CHECK(cards::karo == json("karo")); + + // invalid json -> first enum + CHECK(cards::kreuz == json("what?").get()); + } + + SECTION("traditional enum") + { + // enum -> json + CHECK(json(TS_STOPPED) == "stopped"); + CHECK(json(TS_RUNNING) == "running"); + CHECK(json(TS_COMPLETED) == "completed"); + CHECK(json(TS_INVALID) == json()); + + // json -> enum + CHECK(TS_STOPPED == json("stopped")); + CHECK(TS_RUNNING == json("running")); + CHECK(TS_COMPLETED == json("completed")); + CHECK(TS_INVALID == json()); + + // invalid json -> first enum + CHECK(TS_INVALID == json("what?").get()); + } +} + +enum class strict_cards {kreuz, pik, herz, karo, andere}; // andere not included in mapping + +// NOLINTNEXTLINE(misc-use-internal-linkage,misc-const-correctness) - false positive +NLOHMANN_JSON_SERIALIZE_ENUM_STRICT(strict_cards, +{ + {strict_cards::kreuz, "kreuz"}, + {strict_cards::pik, "pik"}, + {strict_cards::pik, "puk"}, // second entry for cards::pik; will not be used + {strict_cards::herz, "herz"}, + {strict_cards::karo, "karo"} +}) + +enum StrictTaskState // NOLINT(cert-int09-c,readability-enum-initial-value,cppcoreguidelines-use-enum-class) +{ + STRICT_TS_STOPPED, + STRICT_TS_RUNNING, + STRICT_TS_COMPLETED, + STRICT_TS_OTHER, // STRICT_TS_OTHER not in mapping + STRICT_TS_INVALID = -1, +}; + +// NOLINTNEXTLINE(misc-const-correctness,misc-use-internal-linkage) - false positive +NLOHMANN_JSON_SERIALIZE_ENUM_STRICT(StrictTaskState, +{ + {STRICT_TS_INVALID, nullptr}, + {STRICT_TS_STOPPED, "stopped"}, + {STRICT_TS_RUNNING, "running"}, + {STRICT_TS_COMPLETED, "completed"}, +}) + +// regression test for #5708 item 2: NLOHMANN_JSON_SERIALIZE_ENUM_STRICT must not rely on +// unqualified lookup of a helper name that a user's own namespace may also declare +namespace ns_with_colliding_name +{ +// NOLINTNEXTLINE(misc-use-internal-linkage) - used to shadow the library's internal helper name +inline void templated_json_throw(int /*unused*/) {} + +enum class colliding_enum { a, b }; + +// NOLINTNEXTLINE(misc-use-internal-linkage,misc-const-correctness,cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) - false positive +NLOHMANN_JSON_SERIALIZE_ENUM_STRICT(colliding_enum, +{ + {colliding_enum::a, "a"}, + {colliding_enum::b, "b"} +}) +} // namespace ns_with_colliding_name + +TEST_CASE("NLOHMANN_JSON_SERIALIZE_ENUM_STRICT in a namespace with a colliding name") +{ + using ns_with_colliding_name::colliding_enum; + + CHECK(json(colliding_enum::a) == "a"); + CHECK(colliding_enum::b == json("b")); + + json _; + CHECK_THROWS_WITH_AS(_ = json("nope").get(), "[json.exception.out_of_range.410] enum value out of range for colliding_enum: \"nope\"", json::out_of_range&); +} + +TEST_CASE("Strict JSON to enum mapping") +{ + SECTION("enum class") + { + // enum -> json + CHECK(json(strict_cards::kreuz) == "kreuz"); + CHECK(json(strict_cards::pik) == "pik"); + CHECK(json(strict_cards::herz) == "herz"); + CHECK(json(strict_cards::karo) == "karo"); + + // json -> enum + CHECK(json("kreuz").get() == strict_cards::kreuz); + CHECK(json("pik").get() == strict_cards::pik); + CHECK(json("herz").get() == strict_cards::herz); + CHECK(json("karo").get() == strict_cards::karo); + + // comparison of enum and json + CHECK(strict_cards::kreuz == json("kreuz")); + CHECK(strict_cards::pik == json("pik")); + CHECK(strict_cards::herz == json("herz")); + CHECK(strict_cards::karo == json("karo")); + + // invalid json -> exception thrown + json _; + CHECK_THROWS_WITH_AS(_ = json("what?").get(), "[json.exception.out_of_range.410] enum value out of range for strict_cards: \"what?\"", json::out_of_range&); + + // conversion of unmapped enum -> exception thrown + CHECK_THROWS_WITH_AS(json(strict_cards::andere), "[json.exception.out_of_range.410] enum value out of range for strict_cards", json::out_of_range&); + + // comparing an unmapped enum with json throws the same exception + // (the scalar comparison operators used to be noexcept, so this + // called std::terminate) + CHECK_THROWS_WITH_AS(static_cast(strict_cards::andere == json("andere")), "[json.exception.out_of_range.410] enum value out of range for strict_cards", json::out_of_range&); + CHECK_THROWS_WITH_AS(static_cast(json("andere") != strict_cards::andere), "[json.exception.out_of_range.410] enum value out of range for strict_cards", json::out_of_range&); + + // invalid UTF-8 -> out_of_range.410, not the type_error.316 thrown while building the + // message (regression test for #5667); such strings can reach get() unvalidated, + // e.g. from from_cbor()/from_msgpack() (#5529) + const json j_invalid_utf8 = "\xFF"; + CHECK_THROWS_WITH_AS(_ = j_invalid_utf8.get(), "[json.exception.out_of_range.410] enum value out of range for strict_cards: \"\xEF\xBF\xBD\"", json::out_of_range&); + } + + SECTION("traditional enum") + { + // enum -> json + CHECK(json(STRICT_TS_STOPPED) == "stopped"); + CHECK(json(STRICT_TS_RUNNING) == "running"); + CHECK(json(STRICT_TS_COMPLETED) == "completed"); + CHECK(json(STRICT_TS_INVALID) == json()); + + // json -> enum + CHECK(json("stopped").get() == STRICT_TS_STOPPED); + CHECK(json("running").get() == STRICT_TS_RUNNING); + CHECK(json("completed").get() == STRICT_TS_COMPLETED); + CHECK(json().get() == STRICT_TS_INVALID); + + // comparison of enum and json + CHECK(STRICT_TS_STOPPED == json("stopped")); + CHECK(STRICT_TS_RUNNING == json("running")); + CHECK(STRICT_TS_COMPLETED == json("completed")); + CHECK(STRICT_TS_INVALID == json()); + + // invalid json -> exception thrown + json _; + CHECK_THROWS_WITH_AS(_ = json("what?").get(), "[json.exception.out_of_range.410] enum value out of range for StrictTaskState: \"what?\"", json::out_of_range&); + + // conversion of unmapped enum -> exception thrown + CHECK_THROWS_WITH_AS(json(STRICT_TS_OTHER), "[json.exception.out_of_range.410] enum value out of range for StrictTaskState", json::out_of_range&); + + // comparing an unmapped enum with json throws the same exception + CHECK_THROWS_WITH_AS(static_cast(STRICT_TS_OTHER < json("x")), "[json.exception.out_of_range.410] enum value out of range for StrictTaskState", json::out_of_range&); + } +} + + +#ifdef JSON_HAS_CPP_17 +#if JSON_HAS_FILESYSTEM || JSON_HAS_EXPERIMENTAL_FILESYSTEM +TEST_CASE("std::filesystem::path") +{ + SECTION("ascii") + { + json const j_string = "Path"; + auto p = j_string.template get(); + json const j_path = p; + + CHECK(j_path.template get() == + j_string.template get()); + } + + SECTION("utf-8") + { + json const j_string = "P\xc4\x9b\xc5\xa1ina"; + auto p = j_string.template get(); + json const j_path = p; + + CHECK(j_path.template get() == + j_string.template get()); + } +} +#endif + +// the ADL to_json overload for std::u8string only exists under the same guard +// as std::filesystem::path support (it is otherwise only reached indirectly, +// via std::filesystem::path::u8string()) -- mirror both #if conditions from +// include/nlohmann/detail/conversions/to_json.hpp exactly +#if JSON_HAS_FILESYSTEM || JSON_HAS_EXPERIMENTAL_FILESYSTEM +#if defined(__cpp_lib_char8_t) +TEST_CASE("std::u8string") +{ + SECTION("ascii") + { + const std::u8string s = u8"Path"; + json const j = s; + + CHECK(j.template get() == "Path"); + } + + SECTION("utf-8") + { + // use \u universal-character-names (rather than raw \x byte escapes + // or literal non-ASCII source bytes) to compose the multi-byte UTF-8 + // encoding -- MSVC treats \x escapes used that way inside a u8 + // literal as a nonstandard extension (warning C5321), which some of + // our CI configs promote to an error; \u is portable and produces + // the exact same encoded bytes without depending on the source + // file's encoding + const std::u8string s = u8"P\u011B\u0161ina"; + json const j = s; + + CHECK(j.template get() == "P\xc4\x9b\xc5\xa1ina"); + } +} +#endif +#endif + +#if !defined(JSON_NOEXCEPTION) +namespace +{ +// a type whose to_json reports an error by throwing, used below to check that +// converting a std::optional to JSON propagates an exception thrown while +// converting its contained value instead of calling std::terminate (#5642) +struct throwing_to_json_type {}; + +[[noreturn]] void to_json(json& /*unused*/, const throwing_to_json_type& /*unused*/) +{ + throw std::runtime_error("cannot serialize throwing_to_json_type"); +} +} // namespace +#endif + +TEST_CASE("std::optional") +{ + SECTION("null") + { + const json j_null; + const std::optional opt_null; + + CHECK(json(opt_null) == j_null); + CHECK(j_null.get>() == std::nullopt); + + // Constructing std::optional directly from JSON null throws because + // std::optional's own converting constructor is chosen over basic_json's + // operator T(). This is a language-level limitation (std::optional is + // constructible from T, and T is constructible from basic_json via the + // operator); there is no SFINAE path that distinguishes "call from inside + // std::optional's constructor" from "direct call". Use get>() + // or get_to() instead for correct null handling. See #4864 and #5246. + CHECK_THROWS_WITH_AS(std::optional(j_null), + "[json.exception.type_error.302] type must be string, but is null", json::type_error&); + CHECK_THROWS_WITH_AS(std::optional(j_null), + "[json.exception.type_error.302] type must be number, but is null", json::type_error&); + + // Assignment goes through the same overload resolution as direct + // construction, so it throws for the same reason. This relies on + // basic_json's implicit conversion operator, so it only applies + // when JSON_USE_IMPLICIT_CONVERSIONS is enabled (the default). +#if JSON_USE_IMPLICIT_CONVERSIONS + std::optional opt_assign; + CHECK_THROWS_WITH_AS(opt_assign = j_null, + "[json.exception.type_error.302] type must be string, but is null", json::type_error&); +#endif + + // get_to() is the correct way to obtain std::nullopt from a JSON null. + std::optional opt_get_to = "placeholder"; + j_null.get_to(opt_get_to); + CHECK(opt_get_to == std::nullopt); + } + + SECTION("string") + { + json j_string = "string"; + std::optional opt_string = "string"; + + CHECK(json(opt_string) == j_string); + CHECK(std::optional(j_string) == opt_string); + // false positive: Infer attributes the destruction of the temporaries above to opt_string + // @infer-ignore USE_AFTER_DELETE + } + + SECTION("bool") + { + json j_bool = true; + std::optional opt_bool = true; + + CHECK(json(opt_bool) == j_bool); + CHECK(std::optional(j_bool) == opt_bool); + } + + SECTION("number") + { + json j_number = 1; + std::optional opt_int = 1; + + CHECK(json(opt_int) == j_number); + CHECK(j_number.get>() == opt_int); + } + + SECTION("array") + { + json j_array = {1, 2, nullptr}; + std::vector> opt_array = {{1, 2, std::nullopt}}; + + CHECK(json(opt_array) == j_array); + CHECK(j_array.get>>() == opt_array); + } + + SECTION("object") + { + json j_object = {{"one", 1}, {"two", 2}, {"zero", nullptr}}; + std::map> opt_object {{"one", 1}, {"two", 2}, {"zero", std::nullopt}}; + + CHECK(json(opt_object) == j_object); + CHECK(std::map>(j_object) == opt_object); + } + +#if !defined(JSON_NOEXCEPTION) + SECTION("exception from contained value's to_json propagates (#5642)") + { + // to_json(BasicJsonType&, const std::optional&) must not be + // noexcept: it calls T's to_json, which may throw (a user-defined + // to_json that reports an error, or std::bad_alloc for T = + // std::string/vector/json). Before the fix, this called + // std::terminate() instead of letting the exception propagate. + const std::optional opt = throwing_to_json_type{}; + CHECK_THROWS_WITH_AS(json(opt), "cannot serialize throwing_to_json_type", std::runtime_error&); + + // the conversion is noexcept exactly when converting the contained value is + static_assert(!std::is_nothrow_constructible&>::value); + static_assert(std::is_nothrow_constructible&>::value); + } +#endif +} +#endif + +#ifdef JSON_HAS_CPP_17 + #undef JSON_HAS_CPP_17 +#endif + +#ifdef JSON_HAS_CPP_14 + #undef JSON_HAS_CPP_14 +#endif +DOCTEST_CLANG_SUPPRESS_WARNING_POP diff --git a/tests/src/unit-custom-base-class.cpp b/tests/src/unit-custom-base-class.cpp index 38b665793..b9e544705 100644 --- a/tests/src/unit-custom-base-class.cpp +++ b/tests/src/unit-custom-base-class.cpp @@ -505,7 +505,7 @@ static json_with_const_base make_nested_array(std::size_t depth) { if (depth == 0) { - return json_with_const_base(1); + return json_with_const_base(1); // NOLINT(modernize-return-braced-init-list): {1} would be an array } return json_with_const_base::array({make_nested_array(depth - 1)}); } diff --git a/tests/src/unit-disabled_exceptions.cpp b/tests/src/unit-disabled_exceptions.cpp index 8e3adf944..40a37c339 100644 --- a/tests/src/unit-disabled_exceptions.cpp +++ b/tests/src/unit-disabled_exceptions.cpp @@ -14,6 +14,9 @@ DOCTEST_GCC_SUPPRESS_WARNING("-Wnoexcept") #include using json = nlohmann::json; +#ifdef JSON_TEST_NO_GLOBAL_UDLS + using namespace nlohmann::literals; // NOLINT(google-build-using-namespace) +#endif ///////////////////////////////////////////////////////////////////// // for #2824 diff --git a/tests/src/unit-element_access2.cpp b/tests/src/unit-element_access2.cpp index efd7b15a0..9a7a58e6b 100644 --- a/tests/src/unit-element_access2.cpp +++ b/tests/src/unit-element_access2.cpp @@ -1963,8 +1963,8 @@ TEST_CASE("operator[] with user-defined std::string_view-convertible types") }; json j = {{"foo", "from_class"}, {"bar", "from_struct"}}; - TestClass foo_obj; - TestStruct bar_obj; + const TestClass foo_obj; + const TestStruct bar_obj; SECTION("read access") { @@ -2005,6 +2005,10 @@ TEST_CASE("keys convertible to std::string_view work with all lookup functions ( // 3.12.0, such a key worked with at, the const operator[], find, count and // contains via the conversion to std::string; #4958 made the KeyType&& // templates win overload resolution for it instead, and those then failed + // the lookups pick the conversion to std::string_view, which leaves the one + // to std::string unused; it has to exist to reproduce the ambiguity + DOCTEST_CLANG_SUPPRESS_WARNING_PUSH + DOCTEST_CLANG_SUPPRESS_WARNING("-Wunused-member-function") struct DualKey { operator std::string() const @@ -2016,6 +2020,7 @@ TEST_CASE("keys convertible to std::string_view work with all lookup functions ( return "a"; } }; + DOCTEST_CLANG_SUPPRESS_WARNING_POP SECTION("nlohmann::json") { diff --git a/tests/src/unit-modifiers.cpp b/tests/src/unit-modifiers.cpp index 532283f95..3377eace0 100644 --- a/tests/src/unit-modifiers.cpp +++ b/tests/src/unit-modifiers.cpp @@ -1162,6 +1162,7 @@ TEST_CASE("update() on deeply nested values") TEST_CASE("update() with an argument that aliases *this (#5641)") { +#if !defined(JSON_NOEXCEPTION) // checks which exception is thrown, and that nothing changed SECTION("the target is checked before the argument, as before the copy") { json j = 1; @@ -1172,6 +1173,7 @@ TEST_CASE("update() with an argument that aliases *this (#5641)") CHECK_THROWS_WITH_AS(k.update(json::array()), "[json.exception.type_error.312] cannot use update() with array", json::type_error&); CHECK(k == json::object()); } +#endif SECTION("const reference") { diff --git a/tests/src/unit-msgpack.cpp b/tests/src/unit-msgpack.cpp index a92496144..c4b447d64 100644 --- a/tests/src/unit-msgpack.cpp +++ b/tests/src/unit-msgpack.cpp @@ -1560,7 +1560,7 @@ TEST_CASE("MessagePack") // dump() still requires valid UTF-8 and throws for such a value, // unless an error handler that replaces or ignores the bytes is // passed - CHECK_THROWS_AS(j_value.dump(), json::type_error&); + CHECK_THROWS_AS(utils::ignore_return_value(j_value.dump()), json::type_error&); // the same bytes as an object key round-trip as well const std::vector ill_formed_key = {0x81, 0xa2, 0xc0, 0xae, 0x01}; diff --git a/tests/src/unit-ordered_map.cpp b/tests/src/unit-ordered_map.cpp index dce3f61a5..0d0f7730c 100644 --- a/tests/src/unit-ordered_map.cpp +++ b/tests/src/unit-ordered_map.cpp @@ -430,8 +430,8 @@ TEST_CASE("ordered_map") SECTION("with T& (lvalue)") { - std::string one = "1"; - std::string four = "four"; + std::string one = "1"; // NOLINT(misc-const-correctness): emplace must accept a non-const lvalue + std::string four = "four"; // NOLINT(misc-const-correctness): see above auto res1 = om.emplace("eins", one); CHECK(res1.first == om.begin()); @@ -467,7 +467,7 @@ TEST_CASE("ordered_map") SECTION("with key of key_type (non-template overload)") { const std::string key_vier{"vier"}; - std::string four = "four"; + std::string four = "four"; // NOLINT(misc-const-correctness): emplace must accept a non-const lvalue auto res4 = om.emplace(key_vier, four); CHECK(res4.first == om.begin() + 3); diff --git a/tests/src/unit-ubjson.cpp b/tests/src/unit-ubjson.cpp index 450882a12..80ca214a4 100644 --- a/tests/src/unit-ubjson.cpp +++ b/tests/src/unit-ubjson.cpp @@ -2520,7 +2520,7 @@ TEST_CASE("Universal Binary JSON Specification Examples 1") CHECK_NOTHROW(j = json::from_ubjson(v)); REQUIRE(j.is_string()); CHECK(j.get_ref() == std::string("\xc0\xae")); - CHECK_THROWS_AS(j.dump(), json::type_error&); + CHECK_THROWS_AS(utils::ignore_return_value(j.dump()), json::type_error&); CHECK(json::from_ubjson(json::to_ubjson(j)) == j); // the same bytes as an object key round-trip as well diff --git a/tests/src/unit-wstring.cpp b/tests/src/unit-wstring.cpp index c71684002..c555d54da 100644 --- a/tests/src/unit-wstring.cpp +++ b/tests/src/unit-wstring.cpp @@ -37,10 +37,10 @@ TEST_CASE("wide strings") // 32-bit wchar_t first encodes it as an ill-formed three-byte // sequence (rejected one byte later, at column 3) const char* const error_low_surrogate = sizeof(wchar_t) == 2 - ? "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: ill-formed UTF-8 byte; last read: '\"'" + ? "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: ill-formed UTF-8 byte; last read: '\"\xFF'" : "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - invalid string: ill-formed UTF-8 byte; last read: '\"\xED\xB0'"; const char* const error_high_surrogate = sizeof(wchar_t) == 2 - ? "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: ill-formed UTF-8 byte; last read: '\"'" + ? "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: ill-formed UTF-8 byte; last read: '\"\xFF'" : "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - invalid string: ill-formed UTF-8 byte; last read: '\"\xED\xA0'"; // a lone low surrogate cannot start a pair