From 92707f43f2b1a2dffd26b676204c2237f2be2e42 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Nov 2026 19:17:27 +0100 Subject: [PATCH 1/9] build(deps): bump step-security/harden-runner from 2.22.2 to 2.22.3 (#6021) --- .github/workflows/cancel_closed_pr_runs.yml | 2 +- .github/workflows/check_amalgamation.yml | 4 ++-- .github/workflows/check_docs_links.yml | 2 +- .github/workflows/cifuzz.yml | 2 +- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/comment_check_amalgamation.yml | 2 +- .github/workflows/dependency-review.yml | 2 +- .github/workflows/flawfinder.yml | 2 +- .github/workflows/json_view_benchmarks.yml | 2 +- .github/workflows/labeler.yml | 2 +- .github/workflows/publish_documentation.yml | 2 +- .github/workflows/scorecards.yml | 2 +- .github/workflows/semgrep.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/ubuntu.yml | 10 +++++----- 15 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/cancel_closed_pr_runs.yml b/.github/workflows/cancel_closed_pr_runs.yml index 91005de52..fd30c5047 100644 --- a/.github/workflows/cancel_closed_pr_runs.yml +++ b/.github/workflows/cancel_closed_pr_runs.yml @@ -28,7 +28,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit diff --git a/.github/workflows/check_amalgamation.yml b/.github/workflows/check_amalgamation.yml index 253531e40..4fbec781d 100644 --- a/.github/workflows/check_amalgamation.yml +++ b/.github/workflows/check_amalgamation.yml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-26.04 steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit @@ -46,7 +46,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit diff --git a/.github/workflows/check_docs_links.yml b/.github/workflows/check_docs_links.yml index f7e28ab69..0ec9778e6 100644 --- a/.github/workflows/check_docs_links.yml +++ b/.github/workflows/check_docs_links.yml @@ -20,7 +20,7 @@ jobs: timeout-minutes: 30 steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit diff --git a/.github/workflows/cifuzz.yml b/.github/workflows/cifuzz.yml index f5feb9a8f..02a5a8173 100644 --- a/.github/workflows/cifuzz.yml +++ b/.github/workflows/cifuzz.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 61036d79d..99541a96f 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -27,7 +27,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit diff --git a/.github/workflows/comment_check_amalgamation.yml b/.github/workflows/comment_check_amalgamation.yml index ba2fcad7b..558d5fba4 100644 --- a/.github/workflows/comment_check_amalgamation.yml +++ b/.github/workflows/comment_check_amalgamation.yml @@ -20,7 +20,7 @@ jobs: pull-requests: write steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index 27c229572..8034aa1c8 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -21,7 +21,7 @@ jobs: runs-on: ubuntu-26.04 steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit diff --git a/.github/workflows/flawfinder.yml b/.github/workflows/flawfinder.yml index ffc8c2a43..2fae15234 100644 --- a/.github/workflows/flawfinder.yml +++ b/.github/workflows/flawfinder.yml @@ -31,7 +31,7 @@ jobs: security-events: write steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit diff --git a/.github/workflows/json_view_benchmarks.yml b/.github/workflows/json_view_benchmarks.yml index d87357c42..49876a07b 100644 --- a/.github/workflows/json_view_benchmarks.yml +++ b/.github/workflows/json_view_benchmarks.yml @@ -54,7 +54,7 @@ jobs: timeout-minutes: 60 steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 85c7a513d..004039586 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -23,7 +23,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit diff --git a/.github/workflows/publish_documentation.yml b/.github/workflows/publish_documentation.yml index ad53308d7..ba8f435bf 100644 --- a/.github/workflows/publish_documentation.yml +++ b/.github/workflows/publish_documentation.yml @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-26.04 steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index 62e1d504a..47cde1be7 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -40,7 +40,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml index 8b1e6ed5e..ce5c115b4 100644 --- a/.github/workflows/semgrep.yml +++ b/.github/workflows/semgrep.yml @@ -36,7 +36,7 @@ jobs: runs-on: ubuntu-26.04 steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index da7d91fce..b624f9f40 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index c93c0a706..5dd479bcd 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -80,7 +80,7 @@ jobs: runs-on: ubuntu-26.04 steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit @@ -112,7 +112,7 @@ jobs: target: [ci_test_single_header, ci_cppcheck, ci_cpplint, ci_reproducible_tests, ci_non_git_tests, ci_offline_testdata, ci_reuse_compliance, ci_test_valgrind] steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit @@ -178,7 +178,7 @@ jobs: runs-on: ubuntu-26.04 steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit @@ -433,7 +433,7 @@ jobs: runs-on: ubuntu-26.04 steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit @@ -456,7 +456,7 @@ jobs: target: [ci_test_examples, ci_test_build_documentation, ci_test_documentation_mermaid] steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit From 57b898ef9989ea99afc090bc96e435c845993e61 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Tue, 3 Nov 2026 19:17:41 +0100 Subject: [PATCH 2/9] Replace a multibyte decimal point in dump() of a long double (#5998) --- include/nlohmann/detail/output/serializer.hpp | 42 ++++++++++--------- single_include/nlohmann/json.hpp | 42 ++++++++++--------- tests/src/unit-locale-cpp.cpp | 8 ++++ 3 files changed, 54 insertions(+), 38 deletions(-) diff --git a/include/nlohmann/detail/output/serializer.hpp b/include/nlohmann/detail/output/serializer.hpp index 0c004c491..235e7bafd 100644 --- a/include/nlohmann/detail/output/serializer.hpp +++ b/include/nlohmann/detail/output/serializer.hpp @@ -8,7 +8,7 @@ #pragma once -#include // remove, fill, find, none_of, min +#include // copy, none_of, min, search #include // array #include // localeconv, lconv #include // isfinite @@ -2080,31 +2080,35 @@ class serializer // check if the buffer was large enough JSON_ASSERT(static_cast(len) < number_buffer.size()); - // look up the locale's thousands separator and decimal point now, - // matching what snprintf_float() just used (see lexer::get_decimal_point()) + // look up the locale's decimal point now, matching what + // snprintf_float() just used (see get_decimal_point() in + // number_parse.hpp); it is a string, which can be longer than one + // byte (U+066B in ps_AF, #5993). + // A thousands separator needs no handling: %g never groups digits. const auto* loc = std::localeconv(); JSON_ASSERT(loc != nullptr); - const char thousands_sep = (loc->thousands_sep == nullptr) ? '\0' : *loc->thousands_sep; - const char decimal_point = (loc->decimal_point == nullptr) ? '\0' : *loc->decimal_point; - - // erase thousands separators - if (thousands_sep != '\0') - { - // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::remove returns an iterator, see https://github.com/nlohmann/json/issues/3081 - const auto end = std::remove(number_buffer.begin(), number_buffer.begin() + len, thousands_sep); - std::fill(end, number_buffer.end(), '\0'); - JSON_ASSERT((end - number_buffer.begin()) <= len); - len = (end - number_buffer.begin()); - } + const char* const decimal_point = (loc->decimal_point == nullptr) ? "" : loc->decimal_point; + const std::size_t decimal_point_len = std::char_traits::length(decimal_point); // convert decimal point to '.' - if (decimal_point != '\0' && decimal_point != '.') + if ((decimal_point_len > 1) || (decimal_point_len == 1 && decimal_point[0] != '.')) { - // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::find returns an iterator, see https://github.com/nlohmann/json/issues/3081 - const auto dec_pos = std::find(number_buffer.begin(), number_buffer.end(), decimal_point); - if (dec_pos != number_buffer.end()) + const auto first = number_buffer.begin(); + const auto last = first + len; + // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::search returns an iterator, see https://github.com/nlohmann/json/issues/3081 + const auto dec_pos = std::search(first, last, decimal_point, decimal_point + decimal_point_len); + if (dec_pos != last) { *dec_pos = '.'; + if (decimal_point_len > 1) + { + // move the fraction and exponent behind the '.' (std::copy + // allows the overlap, as the destination is before the source) + // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::copy returns an iterator, see https://github.com/nlohmann/json/issues/3081 + const auto end = std::copy(dec_pos + static_cast(decimal_point_len), last, dec_pos + 1); + *end = '\0'; + len = end - first; + } } } diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index cf5a4d711..952cea7b6 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -33446,7 +33446,7 @@ NLOHMANN_JSON_NAMESPACE_END -#include // remove, fill, find, none_of, min +#include // copy, none_of, min, search #include // array #include // localeconv, lconv #include // isfinite @@ -37300,31 +37300,35 @@ class serializer // check if the buffer was large enough JSON_ASSERT(static_cast(len) < number_buffer.size()); - // look up the locale's thousands separator and decimal point now, - // matching what snprintf_float() just used (see lexer::get_decimal_point()) + // look up the locale's decimal point now, matching what + // snprintf_float() just used (see get_decimal_point() in + // number_parse.hpp); it is a string, which can be longer than one + // byte (U+066B in ps_AF, #5993). + // A thousands separator needs no handling: %g never groups digits. const auto* loc = std::localeconv(); JSON_ASSERT(loc != nullptr); - const char thousands_sep = (loc->thousands_sep == nullptr) ? '\0' : *loc->thousands_sep; - const char decimal_point = (loc->decimal_point == nullptr) ? '\0' : *loc->decimal_point; - - // erase thousands separators - if (thousands_sep != '\0') - { - // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::remove returns an iterator, see https://github.com/nlohmann/json/issues/3081 - const auto end = std::remove(number_buffer.begin(), number_buffer.begin() + len, thousands_sep); - std::fill(end, number_buffer.end(), '\0'); - JSON_ASSERT((end - number_buffer.begin()) <= len); - len = (end - number_buffer.begin()); - } + const char* const decimal_point = (loc->decimal_point == nullptr) ? "" : loc->decimal_point; + const std::size_t decimal_point_len = std::char_traits::length(decimal_point); // convert decimal point to '.' - if (decimal_point != '\0' && decimal_point != '.') + if ((decimal_point_len > 1) || (decimal_point_len == 1 && decimal_point[0] != '.')) { - // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::find returns an iterator, see https://github.com/nlohmann/json/issues/3081 - const auto dec_pos = std::find(number_buffer.begin(), number_buffer.end(), decimal_point); - if (dec_pos != number_buffer.end()) + const auto first = number_buffer.begin(); + const auto last = first + len; + // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::search returns an iterator, see https://github.com/nlohmann/json/issues/3081 + const auto dec_pos = std::search(first, last, decimal_point, decimal_point + decimal_point_len); + if (dec_pos != last) { *dec_pos = '.'; + if (decimal_point_len > 1) + { + // move the fraction and exponent behind the '.' (std::copy + // allows the overlap, as the destination is before the source) + // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::copy returns an iterator, see https://github.com/nlohmann/json/issues/3081 + const auto end = std::copy(dec_pos + static_cast(decimal_point_len), last, dec_pos + 1); + *end = '\0'; + len = end - first; + } } } diff --git a/tests/src/unit-locale-cpp.cpp b/tests/src/unit-locale-cpp.cpp index d07eabeff..10143326a 100644 --- a/tests/src/unit-locale-cpp.cpp +++ b/tests/src/unit-locale-cpp.cpp @@ -397,6 +397,14 @@ TEST_CASE("locale with a multi-byte decimal point") // a value the locale-independent paths convert is not affected CHECK(json::parse("12.5") == 12.5); + + // dump() of a long double that takes the snprintf path replaces the + // whole decimal point, not only its first byte (#5993) + CHECK(long_double_json(1.5L).dump() == "1.5"); + CHECK(long_double_json(2.0L).dump() == "2.0"); + CHECK(long_double_json(-0.25L).dump() == "-0.25"); + CHECK(long_double_json::parse(long_double_json(1.5e-7L).dump()) == long_double_json(1.5e-7L)); + CHECK(long_double_json::parse(ld.dump()) == expected_long_double); } if (!tested) { From efc437ef6e25d0e5617c4b6385a095cb383a5a2b Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Wed, 4 Nov 2026 17:13:16 +0100 Subject: [PATCH 3/9] UBJSON/BJData/MessagePack reader: reject a container count equal to SIZE_MAX (#6002) * UBJSON/BJData reader: reject a container count equal to SIZE_MAX The reader marks a container whose count is not known by npos, which is SIZE_MAX. A count of exactly that value was stored unchanged and then read as "no count", so the BJData input [ # M ff ff ff ff ff ff ff ff i 01 ] parsed as [1] instead of throwing out_of_range.408 as a count of 2^64-2 does. Where size_t is 32 bits, the same holds for 'm' and 'L' counts of 2^32-1, in UBJSON too. The counts of 'm', 'M' and the signed types now go through store_ubjson_count(), which rejects npos with "integer value overflow" (the message a count beyond size_t already gets), as the CBOR reader does in get_cbor_container_size(). An ND-array dimension of 2^64-1 gets the same message now, where it was "excessive ndarray size caused overflow" (the product check) on 64-bit platforms. Signed-off-by: Niels Lohmann * MessagePack reader: reject an array/map 32 length equal to SIZE_MAX On a platform with a 32-bit std::size_t, an array 32 or map 32 length of 0xFFFFFFFF equals unknown_size(), so start_array()/start_object() were told the size was unknown and skipped the max_size() check. Route both through the CBOR length check, renamed get_container_size(), which already rejects that value. Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann --- .../nlohmann/detail/input/binary_reader.hpp | 44 ++++++++++++++----- single_include/nlohmann/json.hpp | 44 ++++++++++++++----- tests/src/unit-32bit.cpp | 20 +++++++++ tests/src/unit-bjdata.cpp | 28 ++++++++++-- tests/src/unit-regression2.cpp | 2 +- tests/src/unit-ubjson.cpp | 12 +++++ 6 files changed, 121 insertions(+), 29 deletions(-) diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index 8edc53ae9..b793ae097 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -1024,7 +1024,7 @@ class binary_reader { std::uint64_t len{}; std::size_t size{}; - return get_cbor_argument(len) && get_cbor_container_size(len, size, "array") && enter_cbor_array(size); + return get_cbor_argument(len) && get_container_size(len, size, "array") && enter_cbor_array(size); } case 0x9F: // array (indefinite length) @@ -1064,7 +1064,7 @@ class binary_reader { std::uint64_t len{}; std::size_t size{}; - return get_cbor_argument(len) && get_cbor_container_size(len, size, "map") && enter_object(size); + return get_cbor_argument(len) && get_container_size(len, size, "map") && enter_object(size); } case 0xBF: // map (indefinite length) @@ -1700,7 +1700,7 @@ class binary_reader } /*! - @brief narrow a definite CBOR array/map length to std::size_t + @brief narrow a definite CBOR or MessagePack array/map length to std::size_t A definite length is rejected if it does not fit in std::size_t or if it equals detail::unknown_size(), which is reserved to mark an indefinite- @@ -1713,7 +1713,7 @@ class binary_reader @param[in] context "array" or "map", for the error message @return whether the length is usable */ - bool get_cbor_container_size(const std::uint64_t len, std::size_t& result, const char* context) + bool get_container_size(const std::uint64_t len, std::size_t& result, const char* context) { if (JSON_HEDLEY_UNLIKELY(!value_in_range_of(len) || len == detail::unknown_size())) { @@ -2309,7 +2309,8 @@ class binary_reader case 0xDD: // array 32 { std::uint32_t len{}; - return get_number(len) && enter_msgpack_array(conditional_static_cast(len)); + std::size_t size{}; + return get_number(len) && get_container_size(len, size, "array") && enter_msgpack_array(size); } case 0xDE: // map 16 @@ -2321,7 +2322,8 @@ class binary_reader case 0xDF: // map 32 { std::uint32_t len{}; - return get_number(len) && enter_object(conditional_static_cast(len)); + std::size_t size{}; + return get_number(len) && get_container_size(len, size, "map") && enter_object(size); } // negative fixint @@ -3131,6 +3133,27 @@ class binary_reader return true; } + /*! + @brief store the count of an optimized container (or the length of a string) + + A count equal to npos is rejected: npos marks a container whose count is + not known, so a count of that value would be read as "no count" (compare + get_container_size()). No container can hold that many elements. + + @param[in] count the count read from the input + @param[out] result set to @a count unless it is rejected + @return whether @a count is usable + */ + bool store_ubjson_count(const std::size_t count, std::size_t& result) + { + if (JSON_HEDLEY_UNLIKELY(count == npos)) + { + return out_of_range_error(exception_id::container_too_large, "integer value overflow", "size"); + } + result = count; + return true; + } + /*! @brief read a UBJSON/BJData optimized-container count of a signed marker type ('i', 'I', 'l', 'L') and narrow it to std::size_t @@ -3161,8 +3184,7 @@ class binary_reader { return out_of_range_error(exception_id::container_too_large, "integer value overflow", "size"); } - result = static_cast(number); // NOLINT(bugprone-signed-char-misuse,cert-str34-c): number is not a char - return true; + return store_ubjson_count(static_cast(number), result); // NOLINT(bugprone-signed-char-misuse,cert-str34-c): number is not a char } /*! @@ -3239,8 +3261,7 @@ class binary_reader { return false; } - result = conditional_static_cast(number); - return true; + return store_ubjson_count(conditional_static_cast(number), result); } case 'M': @@ -3258,8 +3279,7 @@ class binary_reader { return out_of_range_error(exception_id::container_too_large, "integer value overflow", "size"); } - result = detail::conditional_static_cast(number); - return true; + return store_ubjson_count(detail::conditional_static_cast(number), result); } case '[': diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 952cea7b6..48c99397b 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -19039,7 +19039,7 @@ class binary_reader { std::uint64_t len{}; std::size_t size{}; - return get_cbor_argument(len) && get_cbor_container_size(len, size, "array") && enter_cbor_array(size); + return get_cbor_argument(len) && get_container_size(len, size, "array") && enter_cbor_array(size); } case 0x9F: // array (indefinite length) @@ -19079,7 +19079,7 @@ class binary_reader { std::uint64_t len{}; std::size_t size{}; - return get_cbor_argument(len) && get_cbor_container_size(len, size, "map") && enter_object(size); + return get_cbor_argument(len) && get_container_size(len, size, "map") && enter_object(size); } case 0xBF: // map (indefinite length) @@ -19715,7 +19715,7 @@ class binary_reader } /*! - @brief narrow a definite CBOR array/map length to std::size_t + @brief narrow a definite CBOR or MessagePack array/map length to std::size_t A definite length is rejected if it does not fit in std::size_t or if it equals detail::unknown_size(), which is reserved to mark an indefinite- @@ -19728,7 +19728,7 @@ class binary_reader @param[in] context "array" or "map", for the error message @return whether the length is usable */ - bool get_cbor_container_size(const std::uint64_t len, std::size_t& result, const char* context) + bool get_container_size(const std::uint64_t len, std::size_t& result, const char* context) { if (JSON_HEDLEY_UNLIKELY(!value_in_range_of(len) || len == detail::unknown_size())) { @@ -20324,7 +20324,8 @@ class binary_reader case 0xDD: // array 32 { std::uint32_t len{}; - return get_number(len) && enter_msgpack_array(conditional_static_cast(len)); + std::size_t size{}; + return get_number(len) && get_container_size(len, size, "array") && enter_msgpack_array(size); } case 0xDE: // map 16 @@ -20336,7 +20337,8 @@ class binary_reader case 0xDF: // map 32 { std::uint32_t len{}; - return get_number(len) && enter_object(conditional_static_cast(len)); + std::size_t size{}; + return get_number(len) && get_container_size(len, size, "map") && enter_object(size); } // negative fixint @@ -21146,6 +21148,27 @@ class binary_reader return true; } + /*! + @brief store the count of an optimized container (or the length of a string) + + A count equal to npos is rejected: npos marks a container whose count is + not known, so a count of that value would be read as "no count" (compare + get_container_size()). No container can hold that many elements. + + @param[in] count the count read from the input + @param[out] result set to @a count unless it is rejected + @return whether @a count is usable + */ + bool store_ubjson_count(const std::size_t count, std::size_t& result) + { + if (JSON_HEDLEY_UNLIKELY(count == npos)) + { + return out_of_range_error(exception_id::container_too_large, "integer value overflow", "size"); + } + result = count; + return true; + } + /*! @brief read a UBJSON/BJData optimized-container count of a signed marker type ('i', 'I', 'l', 'L') and narrow it to std::size_t @@ -21176,8 +21199,7 @@ class binary_reader { return out_of_range_error(exception_id::container_too_large, "integer value overflow", "size"); } - result = static_cast(number); // NOLINT(bugprone-signed-char-misuse,cert-str34-c): number is not a char - return true; + return store_ubjson_count(static_cast(number), result); // NOLINT(bugprone-signed-char-misuse,cert-str34-c): number is not a char } /*! @@ -21254,8 +21276,7 @@ class binary_reader { return false; } - result = conditional_static_cast(number); - return true; + return store_ubjson_count(conditional_static_cast(number), result); } case 'M': @@ -21273,8 +21294,7 @@ class binary_reader { return out_of_range_error(exception_id::container_too_large, "integer value overflow", "size"); } - result = detail::conditional_static_cast(number); - return true; + return store_ubjson_count(detail::conditional_static_cast(number), result); } case '[': diff --git a/tests/src/unit-32bit.cpp b/tests/src/unit-32bit.cpp index 97549c999..e4e2dea6d 100644 --- a/tests/src/unit-32bit.cpp +++ b/tests/src/unit-32bit.cpp @@ -97,3 +97,23 @@ TEST_CASE("CBOR") } } } + +TEST_CASE("MessagePack") +{ + SECTION("parse errors") + { + SECTION("array/map size equal to the unknown-size sentinel") + { + // on 32-bit platforms an array 32/map 32 length of 0xFFFFFFFF aliases unknown_size() + std::vector const varr = {0xDD, 0xFF, 0xFF, 0xFF, 0xFF}; + std::vector const vmap = {0xDF, 0xFF, 0xFF, 0xFF, 0xFF}; + + json _; + CHECK_THROWS_WITH_AS(_ = json::from_msgpack(varr), "[json.exception.out_of_range.408] syntax error while parsing MessagePack size: excessive array size", json::out_of_range&); + CHECK(json::from_msgpack(varr, true, false).is_discarded()); + + CHECK_THROWS_WITH_AS(_ = json::from_msgpack(vmap), "[json.exception.out_of_range.408] syntax error while parsing MessagePack size: excessive map size", json::out_of_range&); + CHECK(json::from_msgpack(vmap, true, false).is_discarded()); + } + } +} diff --git a/tests/src/unit-bjdata.cpp b/tests/src/unit-bjdata.cpp index ec7faa6cd..ecc79959d 100644 --- a/tests/src/unit-bjdata.cpp +++ b/tests/src/unit-bjdata.cpp @@ -3237,14 +3237,34 @@ TEST_CASE("BJData") #endif CHECK(json::from_bjdata(vM, true, false).is_discarded()); -#if SIZE_MAX != 0xffffffff - CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vMX), "[json.exception.out_of_range.408] syntax error while parsing BJData size: excessive ndarray size caused overflow", json::out_of_range&); -#else + // the first dimension is 2^64-1, which is rejected as a count (see below) CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vMX), "[json.exception.out_of_range.408] syntax error while parsing BJData size: integer value overflow", json::out_of_range&); -#endif CHECK(json::from_bjdata(vMX, true, false).is_discarded()); } + SECTION("optimized array: count equal to SIZE_MAX") + { + // the reader marks a container without a count by SIZE_MAX, so + // a count of that value must not be read as "no count" + // (it used to parse as [1]) + json _; +#if SIZE_MAX != 0xffffffff + std::vector const vM = {'[', '#', 'M', 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 'i', 0x01, ']'}; + CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vM), "[json.exception.out_of_range.408] syntax error while parsing BJData size: integer value overflow", json::out_of_range&); + CHECK(json::from_bjdata(vM, true, false).is_discarded()); + // one less is an ordinary excessive count + std::vector const vM1 = {'[', '#', 'M', 0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 'i', 0x01, ']'}; + CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vM1), "[json.exception.out_of_range.408] excessive array size: 18446744073709551614", json::out_of_range&); +#else + std::vector const vm = {'[', '#', 'm', 0xFF, 0xFF, 0xFF, 0xFF, 'i', 0x01, ']'}; + CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vm), "[json.exception.out_of_range.408] syntax error while parsing BJData size: integer value overflow", json::out_of_range&); + CHECK(json::from_bjdata(vm, true, false).is_discarded()); + std::vector const vL = {'[', '#', 'L', 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 'i', 0x01, ']'}; + CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vL), "[json.exception.out_of_range.408] syntax error while parsing BJData size: integer value overflow", json::out_of_range&); + CHECK(json::from_bjdata(vL, true, false).is_discarded()); +#endif + } + SECTION("optimized array: integer value overflow") { #if SIZE_MAX == 0xffffffff diff --git a/tests/src/unit-regression2.cpp b/tests/src/unit-regression2.cpp index 08d342d0a..f7bd557e8 100644 --- a/tests/src/unit-regression2.cpp +++ b/tests/src/unit-regression2.cpp @@ -1009,7 +1009,7 @@ TEST_CASE("regression test - excessive binary container size honors allow_except // allow_exceptions=true (the default) must still throw exactly as before. // The exact message text is not checked here: on platforms where // std::size_t is 32-bit, the CBOR reader's own length-narrowing check - // (get_cbor_container_size(), unrelated to this fix) intercepts a + // (get_container_size(), unrelated to this fix) intercepts a // declared length of 2^63 before it ever reaches the check this test // targets, with different (but equally valid, and already correct) // wording -- see unit-cbor.cpp for coverage of that message. diff --git a/tests/src/unit-ubjson.cpp b/tests/src/unit-ubjson.cpp index 4bc7f7ac0..b4fdc0997 100644 --- a/tests/src/unit-ubjson.cpp +++ b/tests/src/unit-ubjson.cpp @@ -2385,6 +2385,18 @@ TEST_CASE("UBJSON optimized arrays of a valueless type are bounded") } } + SECTION("a count equal to SIZE_MAX is rejected") + { + // the reader marks a container without a count by SIZE_MAX, so a + // count of that value must not be read as "no count"; UBJSON's + // largest count type (int64) can only reach it where size_t is 32 bits +#if SIZE_MAX == 0xffffffff + const std::vector input = {'[', '#', 'L', 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, 'i', 0x01, ']'}; + CHECK_THROWS_WITH_AS(_ = json::from_ubjson(input), "[json.exception.out_of_range.408] syntax error while parsing UBJSON size: integer value overflow", json::out_of_range&); + CHECK(json::from_ubjson(input, true, false).is_discarded()); +#endif + } + SECTION("ordinary counts are unaffected") { CHECK(json::from_ubjson(std::vector({'[', '$', 'Z', '#', 'i', 3})) == json({nullptr, nullptr, nullptr})); From 42825fba0744d0169c59a18c4bca6f8e3155be20 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Wed, 4 Nov 2026 17:13:25 +0100 Subject: [PATCH 4/9] Binary writers: zero only a window ahead of the vector sink's position (#6020) output_vector_sink grew the vector by resizing it to its whole new capacity. resize() zeroes the characters it adds, so every doubling zeroed megabytes of a fresh allocation at once: that faults in all its pages and evicts them from the cache before the writer reaches them, and the writer's stores then miss again. The sink now reserves the same capacity as before but resizes the vector only to 64 KiB (vector_sink_window) ahead of the position, which is still cached when the writer's stores come. to_cbor(), to_msgpack(), to_ubjson(), to_bjdata() and to_bon8() of an array of one million numbers into a fresh vector: 4-26 % faster on x86 (Skylake laptop with Apple clang 13, Haswell VPS with GCC 16 and Clang 23), 1-2.5 % on Apple M1 Max; other documents unchanged (within 0.4 % in reruns). The capacity and growth of the vector are as before. Signed-off-by: Niels Lohmann --- .../detail/output/output_adapters.hpp | 21 +++++++++++++++---- single_include/nlohmann/json.hpp | 21 +++++++++++++++---- tests/src/unit-binary_writer_sinks.cpp | 10 ++++++++- 3 files changed, 43 insertions(+), 9 deletions(-) diff --git a/include/nlohmann/detail/output/output_adapters.hpp b/include/nlohmann/detail/output/output_adapters.hpp index e9e57ab9a..b1ea305b9 100644 --- a/include/nlohmann/detail/output/output_adapters.hpp +++ b/include/nlohmann/detail/output/output_adapters.hpp @@ -116,6 +116,10 @@ inline void copy_short(CharType* dst, const CharType* src, const std::size_t len /// the number of characters an empty vector is grown to by output_vector_sink constexpr std::size_t min_vector_sink_capacity = 64; +/// the number of characters output_vector_sink makes writable ahead of its +/// position at a time: 64 KiB of the vector's capacity, not all of it +constexpr std::size_t vector_sink_window = 65536; + /// @brief non-virtual output sink writing into a std::vector /// /// This sink is not part of the virtual output_adapter_protocol hierarchy: it is @@ -129,7 +133,8 @@ constexpr std::size_t min_vector_sink_capacity = 64; /// encoding them: insert() is an out-of-line call into a routine that can insert /// anywhere, and resize() is one as well with libstdc++. The sink therefore keeps /// its own write position: it grows the vector geometrically ahead of the -/// position and copies each write to it, so a write is a capacity check and a +/// position (making @ref vector_sink_window characters of it writable at a +/// time) and copies each write to it, so a write is a capacity check and a /// short copy. The destructor shrinks the vector to the bytes actually written, /// also when a write throws; shrinking a vector never allocates. The sink owns /// that trim, so it can be moved but not copied, and the vector must not be used @@ -277,9 +282,17 @@ class output_vector_sink // reallocate for every few bytes const std::size_t written = position(); const std::size_t needed = written + length; - v->resize((std::max)(min_vector_sink_capacity, needed <= v->capacity() - ? v->capacity() - : (std::max)(needed, v->capacity() * 2))); + const std::size_t capacity = (std::max)(min_vector_sink_capacity, needed <= v->capacity() + ? v->capacity() + : (std::max)(needed, v->capacity() * 2)); + v->reserve(capacity); + // resize() zeroes the characters it adds. Zeroing all of a doubled + // capacity, megabytes of a fresh allocation, at once faults in its + // pages and evicts them from the cache before the writer gets there, + // so its stores miss again; a window ahead of the position is still + // cached when they come (the binary writers on a long array of + // numbers: 4-26 % faster on x86, 1-2.5 % on Apple M1) + v->resize((std::min)(capacity, (std::max)(needed, written + vector_sink_window))); cursor = v->data() + written; limit = v->data() + v->size(); } diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 48c99397b..bb81909ae 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -27511,6 +27511,10 @@ inline void copy_short(CharType* dst, const CharType* src, const std::size_t len /// the number of characters an empty vector is grown to by output_vector_sink constexpr std::size_t min_vector_sink_capacity = 64; +/// the number of characters output_vector_sink makes writable ahead of its +/// position at a time: 64 KiB of the vector's capacity, not all of it +constexpr std::size_t vector_sink_window = 65536; + /// @brief non-virtual output sink writing into a std::vector /// /// This sink is not part of the virtual output_adapter_protocol hierarchy: it is @@ -27524,7 +27528,8 @@ constexpr std::size_t min_vector_sink_capacity = 64; /// encoding them: insert() is an out-of-line call into a routine that can insert /// anywhere, and resize() is one as well with libstdc++. The sink therefore keeps /// its own write position: it grows the vector geometrically ahead of the -/// position and copies each write to it, so a write is a capacity check and a +/// position (making @ref vector_sink_window characters of it writable at a +/// time) and copies each write to it, so a write is a capacity check and a /// short copy. The destructor shrinks the vector to the bytes actually written, /// also when a write throws; shrinking a vector never allocates. The sink owns /// that trim, so it can be moved but not copied, and the vector must not be used @@ -27672,9 +27677,17 @@ class output_vector_sink // reallocate for every few bytes const std::size_t written = position(); const std::size_t needed = written + length; - v->resize((std::max)(min_vector_sink_capacity, needed <= v->capacity() - ? v->capacity() - : (std::max)(needed, v->capacity() * 2))); + const std::size_t capacity = (std::max)(min_vector_sink_capacity, needed <= v->capacity() + ? v->capacity() + : (std::max)(needed, v->capacity() * 2)); + v->reserve(capacity); + // resize() zeroes the characters it adds. Zeroing all of a doubled + // capacity, megabytes of a fresh allocation, at once faults in its + // pages and evicts them from the cache before the writer gets there, + // so its stores miss again; a window ahead of the position is still + // cached when they come (the binary writers on a long array of + // numbers: 4-26 % faster on x86, 1-2.5 % on Apple M1) + v->resize((std::min)(capacity, (std::max)(needed, written + vector_sink_window))); cursor = v->data() + written; limit = v->data() + v->size(); } diff --git a/tests/src/unit-binary_writer_sinks.cpp b/tests/src/unit-binary_writer_sinks.cpp index 7ec2eb8fc..5638435ca 100644 --- a/tests/src/unit-binary_writer_sinks.cpp +++ b/tests/src/unit-binary_writer_sinks.cpp @@ -50,6 +50,14 @@ std::vector test_values() large_payloads.push_back(json::binary(std::vector(static_cast(100 * i), static_cast(i)))); } + // several times vector_sink_window characters in every format: the vector + // sink makes the capacity it grows to writable a window at a time + json many_windows = json::array(); + for (std::uint64_t i = 0; i < 30000; ++i) + { + many_windows.push_back((i * 0x9E3779B97F4A7C15u) >> 1u); + } + return { json(nullptr), json(true), json(false), @@ -61,7 +69,7 @@ std::vector test_values() json::array(), json::object(), json::array({1, 2, 3}), json({{"a", 1}, {"b", nullptr}}), json({{"nested", {{"deep", json::array({1, "two", 3.0, nullptr})}}}}), - big_array, big_object, large_payloads, + big_array, big_object, large_payloads, many_windows, json::binary(std::vector(5000, 0x42)) }; } From a49242f0880b5e1163ac89a2163c787ad6aca0eb Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Wed, 4 Nov 2026 17:46:20 +0100 Subject: [PATCH 5/9] Docs: reading and writing files; patch_inplace() is not atomic (#6017) --- .../docs/api/basic_json/patch_inplace.md | 19 ++- docs/mkdocs/docs/features/files.md | 130 ++++++++++++++++++ docs/mkdocs/docs/features/index.md | 2 + docs/mkdocs/docs/features/json_patch.md | 3 +- docs/mkdocs/docs/features/json_view.md | 4 +- docs/mkdocs/docs/features/parsing/index.md | 5 + docs/mkdocs/docs/features/performance.md | 4 +- docs/mkdocs/docs/features/serialization.md | 3 + docs/mkdocs/docs/home/faq.md | 2 +- docs/mkdocs/mkdocs.yml | 1 + 10 files changed, 166 insertions(+), 7 deletions(-) create mode 100644 docs/mkdocs/docs/features/files.md diff --git a/docs/mkdocs/docs/api/basic_json/patch_inplace.md b/docs/mkdocs/docs/api/basic_json/patch_inplace.md index ff65e8dbd..c3cd4f530 100644 --- a/docs/mkdocs/docs/api/basic_json/patch_inplace.md +++ b/docs/mkdocs/docs/api/basic_json/patch_inplace.md @@ -15,7 +15,9 @@ the patch. This function applies a JSON patch in place and returns void. ## Exception safety -No guarantees, value may be corrupted by an unsuccessful patch operation. +No guarantees: if an operation fails, the operations before it stay applied, and a failing `move` operation may already +have removed the value at its "from" location. Use [`patch`](patch.md) if the value must be unchanged after a failed +patch. ## Exceptions @@ -62,8 +64,19 @@ the JSON value, the complexity can usually be neglected. ## Notes Unlike [`patch`](patch.md), `patch_inplace` applies the operation "in place" and no copy of the JSON value is created. -That makes it faster for large documents by avoiding the copy. However, the JSON value might be corrupted if the -function throws an exception. +That makes it faster for large documents by avoiding the copy. + +!!! warning "Not atomic" + + `patch_inplace` applies the operations one after the other and does not undo them when one fails. After an + exception, the value holds the result of the operations before the failing one, and a failing `move` operation + (a "remove" followed by an "add") may already have removed the value at its "from" location; see the second example + below. [RFC 6902](https://tools.ietf.org/html/rfc6902#section-5) requires that a patch with a failing operation is + not deemed successful, but leaves open what happens to the document. + + If the value must stay unchanged when the patch fails, use [`patch`](patch.md), which applies the patch to a copy: + `#!cpp j = j.patch(json_patch);`. Validating the patch beforehand is not enough, as "test" operations and missing + paths are only detected while the patch is applied. Duplicate member names inside an operation object are resolved by the parser before the patch is applied (the last one is kept, see [`object_t`](object_t.md#behavior)), so `#!json {"op": "add", "op": "remove", "path": "/a"}` is a diff --git a/docs/mkdocs/docs/features/files.md b/docs/mkdocs/docs/features/files.md new file mode 100644 index 000000000..3833f13db --- /dev/null +++ b/docs/mkdocs/docs/features/files.md @@ -0,0 +1,130 @@ +# Reading and writing files + +The library has no functions that take a file name. It reads from and writes to streams, so files are handled with +`#!cpp std::ifstream` and `#!cpp std::ofstream` (or a `#!cpp FILE*` when reading). This page collects the recipes for +the common cases and the mistakes that show up again and again. + +## Reading a JSON file + +Pass the stream to [`parse`](../api/basic_json/parse.md) directly, and check that the file was opened: + +```cpp +#include +#include +#include +#include + +using json = nlohmann::json; + +json read_json_file(const std::string& path) +{ + std::ifstream f(path); + if (!f.is_open()) + { + throw std::runtime_error("cannot open " + path); + } + return json::parse(f); +} +``` + +- **Check that the file is open.** A stream that failed to open looks like an empty input, so `parse` reports + [`parse_error.101`](../home/exceptions.md#jsonexceptionparse_error101) ("unexpected end of input") at line 1, + column 1 instead of saying that the file is missing. A relative path is resolved against the working directory of the + running program, which is often not the directory of the source file (see the + [FAQ](../home/faq.md#parsing-a-file-fails-with-unexpected-end-of-input)). +- **Don't read the file into a string first.** Parsing from a `#!cpp std::ifstream` is only about 10 to 20 % slower + than reading the whole file into a string with one `read()` and parsing that, and it needs no memory for the copy (see + [input sources and performance](parsing/index.md#input-sources-and-performance)). The common way to read a file with + `#!cpp std::istreambuf_iterator` is so slow that it takes about twice as long as parsing from the stream. Read + the file into a string only if you need the text itself, for example for an error message. This advice is for + `#!cpp json::parse`. [`json_document`](json_view.md) reads a stream into a buffer that it owns anyway, so there is no + copy to save: reading the file into a `#!cpp std::string` yourself and passing that as an lvalue, which the document + borrows, was about 6 to 12 % faster than parsing from the stream, with the same single buffer. +- `parse` requires the file to hold exactly one JSON value. Files with one value per line are read with + [JSON Lines](parsing/json_lines.md); files too large for memory are covered in + [Large, streamed, and partial input](parsing/large_and_partial_input.md). + +## Writing a JSON file + +[`dump`](../api/basic_json/dump.md) the value into a string first, then write the string: + +```cpp +void write_json_file(const std::string& path, const json& j) +{ + const std::string text = j.dump(4) + '\n'; // may throw: nothing is written yet + + std::ofstream f(path, std::ios::binary); + if (!f.is_open()) + { + throw std::runtime_error("cannot open " + path); + } + f << text; + f.close(); + if (f.fail()) + { + throw std::runtime_error("cannot write " + path); + } +} +``` + +- **Serialize before opening the file.** Opening a `#!cpp std::ofstream` truncates the file. `dump` and + [`operator<<`](../api/operator_ltlt.md) throw [`type_error.316`](../home/exceptions.md#jsonexceptiontype_error316) + when a string is not valid UTF-8 (see [handling invalid UTF-8](serialization.md#handling-invalid-utf-8)), and + `#!cpp f << std::setw(4) << j` may already have written part of the value when it throws. Serializing into a string + first leaves the old file untouched in that case. +- **Check the stream after closing it.** A full disk or a missing permission sets the stream's error state; it does not + throw unless you enabled exceptions on the stream with `exceptions()`. +- `std::ios::binary` writes the text as it is. Without it, Windows writes `\r\n` for every newline of a pretty-printed + value. + +## Replacing a file without corrupting it + +The library never writes a file itself, so this section is not about the library. It is general C++ advice for a +problem users of the library keep running into. + +A program that crashes or is killed while it writes leaves a truncated file behind. To make sure readers see either the +old or the new contents, write to a temporary file in the same directory and rename it over the original +(C++17, `#!cpp `): + +```cpp +#include + +void replace_json_file(const std::filesystem::path& path, const json& j) +{ + std::filesystem::path tmp = path; + tmp += ".tmp"; // use a unique name if several processes may write the file + + write_json_file(tmp.string(), j); + std::filesystem::rename(tmp, path); // replaces path +} +``` + +The temporary file must be in the same directory (more precisely, on the same file system) as the target: a rename +across file systems does not fall back to a copy, it fails and throws `#!cpp std::filesystem::filesystem_error`. On +POSIX systems, the rename atomically replaces the file. Standard C++ has no way to flush a file to the disk (`fsync`), +so this protects against a crashed program, but not necessarily against a power loss. + +## Binary formats + +The [binary formats](binary_formats/index.md) work the same way, with the streams opened in binary mode: + +```cpp +// read +std::ifstream in("data.cbor", std::ios::binary); +json j = json::from_cbor(in); + +// write +const std::vector bytes = json::to_cbor(j); +std::ofstream out("data.cbor", std::ios::binary); +out.write(reinterpret_cast(bytes.data()), static_cast(bytes.size())); +``` + +Without `std::ios::binary`, Windows turns the bytes `0x0D 0x0A` into `0x0A` and may stop at a byte `0x1A` while reading, +and turns `0x0A` into `0x0D 0x0A` while writing, which corrupts the data. + +## See also + +- [Parsing](parsing/index.md) - the inputs `parse` accepts +- [Serialization](serialization.md) - the options of `dump` +- [`parse`](../api/basic_json/parse.md), [`dump`](../api/basic_json/dump.md) +- [`operator>>`](../api/operator_gtgt.md), [`operator<<`](../api/operator_ltlt.md) diff --git a/docs/mkdocs/docs/features/index.md b/docs/mkdocs/docs/features/index.md index f9b0f20cc..d8c6c9367 100644 --- a/docs/mkdocs/docs/features/index.md +++ b/docs/mkdocs/docs/features/index.md @@ -46,6 +46,8 @@ C++ types, and finally serialize it again. [BJData](binary_formats/bjdata.md), [BON8](binary_formats/bon8.md), [BSON](binary_formats/bson.md), [CBOR](binary_formats/cbor.md), [MessagePack](binary_formats/messagepack.md), or [UBJSON](binary_formats/ubjson.md). - [Binary values](binary_values.md) — store and exchange raw byte sequences. +- [Reading and writing files](files.md) — read and write files safely, including replacing a file without + corrupting it. ## How values are stored and configured diff --git a/docs/mkdocs/docs/features/json_patch.md b/docs/mkdocs/docs/features/json_patch.md index ea22b4e15..d9e96c831 100644 --- a/docs/mkdocs/docs/features/json_patch.md +++ b/docs/mkdocs/docs/features/json_patch.md @@ -29,7 +29,8 @@ other three continue on its result. [`patch`](../api/basic_json/patch.md) leaves the original value unchanged and returns the patched result as a copy. If the document is large and the original value is no longer needed, - [`patch_inplace`](../api/basic_json/patch_inplace.md) applies the same operations in place instead. + [`patch_inplace`](../api/basic_json/patch_inplace.md) applies the same operations in place instead. Unlike + `patch`, it is not atomic: if an operation fails, the operations before it stay applied. !!! note "Duplicate member names" diff --git a/docs/mkdocs/docs/features/json_view.md b/docs/mkdocs/docs/features/json_view.md index 44bf581a5..5ab2281f0 100644 --- a/docs/mkdocs/docs/features/json_view.md +++ b/docs/mkdocs/docs/features/json_view.md @@ -160,7 +160,9 @@ document: `#!cpp auto v = json_document::parse(text).root();` does not compile. wide. The limit is 16 bytes below 2^32, because the parser reads in 16-byte steps, and a position one step past the end must still fit. - **A stream is always read to its end.** There is no partial/streaming read of an `#!cpp std::istream`: the stream - is read to its end into a buffer the document owns, and parsing starts after that. + is read to its end into a buffer the document owns, and parsing starts after that. For a file, reading it into a + `#!cpp std::string` yourself and parsing that (an lvalue is borrowed) is therefore not more expensive in memory, and + was about 6 to 12 % faster than parsing from an `#!cpp std::ifstream`. - **Fewer source positions after edits.** With [`JSON_DIAGNOSTIC_POSITIONS`](../api/macros/json_diagnostic_positions.md) enabled, [`materialize()`](../api/basic_json_view/materialize.md) sets the same positions as `#!cpp basic_json::parse()`. diff --git a/docs/mkdocs/docs/features/parsing/index.md b/docs/mkdocs/docs/features/parsing/index.md index 901626952..93a02b501 100644 --- a/docs/mkdocs/docs/features/parsing/index.md +++ b/docs/mkdocs/docs/features/parsing/index.md @@ -35,6 +35,8 @@ std::ifstream f("example.json"); json data = json::parse(f); ``` +See [Reading and writing files](../files.md) for checking that the file was opened and for writing files safely. + The input must be encoded in UTF-8; other encodings are not supported. The exception is wide-character input, which is read as UTF-16 or UTF-32 — depending on the character type — and converted to UTF-8; see the [`parse`](../../api/basic_json/parse.md) page for an example. A single input may contain only one JSON value. @@ -84,6 +86,9 @@ ahead: after parsing, it is positioned exactly as if every character had been re [`operator>>`](../../api/operator_gtgt.md) can still read several values from one stream (see also [JSON Lines](json_lines.md)). +This does not hold for [`json_document`](../json_view.md): it reads a stream into a buffer it owns, so the stream saves +no memory, and reading the file into a `#!cpp std::string` yourself and borrowing it is about 6 to 12 % faster. + Two macros switch these optimizations off without changing any result: [`JSON_DISABLE_STREAM_BULK_SCAN`](../../api/macros/json_disable_stream_bulk_scan.md) for stream buffers that do not follow the usual `std::streambuf` contract, and diff --git a/docs/mkdocs/docs/features/performance.md b/docs/mkdocs/docs/features/performance.md index ffea9c215..8f4508057 100644 --- a/docs/mkdocs/docs/features/performance.md +++ b/docs/mkdocs/docs/features/performance.md @@ -39,7 +39,9 @@ iterator pair over non-contiguous but random-access storage (e.g. `#!cpp std::de optimization but not the second, since the byte-scanning fast path requires contiguous storage. Practically: a file can be passed directly to `parse`, which also avoids the memory for a copy of the file. Reading it -into a string first is no longer necessary for speed: +into a string first is no longer necessary for speed (this is for `#!cpp json::parse`; a +[`json_document`](json_view.md) reads a stream into an owned buffer anyway, so for it reading the file into a string +and borrowing that is the faster way): ```cpp // fast: the stream buffer's get area is scanned in bulk diff --git a/docs/mkdocs/docs/features/serialization.md b/docs/mkdocs/docs/features/serialization.md index c1b23442c..fd482cb73 100644 --- a/docs/mkdocs/docs/features/serialization.md +++ b/docs/mkdocs/docs/features/serialization.md @@ -17,6 +17,9 @@ To write a value directly to a stream (for example, a file or `#!cpp std::cout`) std::cout << j << std::endl; ``` +To write a file without leaving a truncated file behind when serialization fails, see +[Reading and writing files](files.md#writing-a-json-file). + !!! note "String, not raw value" `dump` always returns a **JSON text**. Serializing a JSON string therefore includes the surrounding quotes and diff --git a/docs/mkdocs/docs/home/faq.md b/docs/mkdocs/docs/home/faq.md index c43acef37..d06819569 100644 --- a/docs/mkdocs/docs/home/faq.md +++ b/docs/mkdocs/docs/home/faq.md @@ -110,7 +110,7 @@ in the source files that need it, instantiate `basic_json` once, and consider pr The parser did not see any character: the file was most likely not opened. A relative path is resolved against the working directory of the running program, which is often the build directory of your IDE, not the source directory. Check `is_open()` and try an absolute path. Other causes are an empty file and a stream that was already read. See -[`parse_error.101`](exceptions.md#jsonexceptionparse_error101). +[`parse_error.101`](exceptions.md#jsonexceptionparse_error101) and [Reading and writing files](../features/files.md). ### Input that arrives in chunks diff --git a/docs/mkdocs/mkdocs.yml b/docs/mkdocs/mkdocs.yml index 08c46ff83..ce5cdc226 100644 --- a/docs/mkdocs/mkdocs.yml +++ b/docs/mkdocs/mkdocs.yml @@ -130,6 +130,7 @@ nav: - features/element_access/unchecked_access.md - features/element_access/checked_access.md - features/element_access/default_value.md + - features/files.md - features/iterators.md - features/merge_patch.md - features/json_patch.md From 1da873515416639583b7c6143646bbf064431657 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20Gr=C3=BCninger?= Date: Wed, 4 Nov 2026 23:38:47 +0100 Subject: [PATCH 6/9] Fix leak and crash when an allocation throws during parsing (#6023) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit json_sax_dom_parser::insert_key_with_index: own the new object_key_index in a unique_ptr before key_indexes may reallocate; a throwing reallocation leaked it. Signed-off-by: Christoph Grüninger --- include/nlohmann/detail/input/json_sax.hpp | 3 ++- single_include/nlohmann/json.hpp | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/include/nlohmann/detail/input/json_sax.hpp b/include/nlohmann/detail/input/json_sax.hpp index 80850ab92..fc9e9b774 100644 --- a/include/nlohmann/detail/input/json_sax.hpp +++ b/include/nlohmann/detail/input/json_sax.hpp @@ -904,7 +904,8 @@ class json_sax_dom_parser : public json_sax_dom_error_state { hashes.push_back(hasher(element.first)); } - ordered_data.key_indexes.emplace_back(new object_key_index(std::move(hashes))); + std::unique_ptr index(new object_key_index(std::move(hashes))); // NOLINT(cppcoreguidelines-owning-memory,modernize-make-unique): std::make_unique is C++14 + ordered_data.key_indexes.push_back(std::move(index)); f.ordered.index = ordered_data.key_indexes.back().get(); } diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index bb81909ae..50089514f 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -16538,7 +16538,8 @@ class json_sax_dom_parser : public json_sax_dom_error_state { hashes.push_back(hasher(element.first)); } - ordered_data.key_indexes.emplace_back(new object_key_index(std::move(hashes))); + std::unique_ptr index(new object_key_index(std::move(hashes))); // NOLINT(cppcoreguidelines-owning-memory,modernize-make-unique): std::make_unique is C++14 + ordered_data.key_indexes.push_back(std::move(index)); f.ordered.index = ordered_data.key_indexes.back().get(); } From 20fe34ceeb58a3c1ba3750258c56fb231d778762 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 4 Nov 2026 23:38:57 +0100 Subject: [PATCH 7/9] build(deps): bump lukka/get-cmake from 4.4.4 to 4.4.5 (#6024) Bumps [lukka/get-cmake](https://github.com/lukka/get-cmake) from 4.4.4 to 4.4.5. - [Release notes](https://github.com/lukka/get-cmake/releases) - [Changelog](https://github.com/lukka/get-cmake/blob/main/RELEASE_PROCESS.md) - [Commits](https://github.com/lukka/get-cmake/compare/a89e43e22a96abafcd7a62db652a65ecd33052c9...f86dd60acae0d132056913e248bdabf0e0580b7a) --- updated-dependencies: - dependency-name: lukka/get-cmake dependency-version: 4.4.5 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ubuntu.yml | 36 +++++++++++++++++------------------ .github/workflows/windows.yml | 4 ++-- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 5dd479bcd..581fb6081 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -25,7 +25,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -38,7 +38,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Check that Meson and pkg-config offer the CMake options run: make check_build_options - name: Install Meson @@ -92,7 +92,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -123,7 +123,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -142,7 +142,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -168,7 +168,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -244,7 +244,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Run CMake run: CXX=g++-${{ matrix.compiler }} cmake -S . -B build -DJSON_CI=On - name: Build @@ -262,7 +262,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -279,7 +279,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Disable filesystem support (std::filesystem is unusable) for clang 7 / 8 / 9 / 10 run: echo "JSON_FORCED_GLOBAL_COMPILE_OPTIONS=-DJSON_HAS_FILESYSTEM=0;-DJSON_HAS_EXPERIMENTAL_FILESYSTEM=0" >> "$GITHUB_ENV" if: ${{ matrix.compiler == '7' || matrix.compiler == '8' || matrix.compiler == '9' || matrix.compiler == '10' }} @@ -299,7 +299,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -319,7 +319,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build with libc++ @@ -346,7 +346,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -366,7 +366,7 @@ jobs: # import-std support. Its opt-in token is CMake-version-specific, so pin # CMake to the version whose token is set in tests/module_cpp20/CMakeLists.txt. - name: Get pinned CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 with: cmakeVersion: 4.3.4 # Clang: the std library module is provided by libc++ (the image's libstdc++ @@ -392,7 +392,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -409,7 +409,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -423,7 +423,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -443,7 +443,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Run CMake run: cmake -S . -B build -DCMAKE_TOOLCHAIN_FILE=$EMSDK/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake -GNinja - name: Build @@ -484,7 +484,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Check that the job runs natively on arm64 run: uname -m && test "$(uname -m)" = aarch64 - name: Run CMake diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 59647ed94..b9f9d0251 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -90,7 +90,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Set extra CXX_FLAGS for latest std_version # /wd5285 silences C5285 emitted by the bundled third-party doctest.h, which # specializes std::tuple (newly diagnosed by the VS2026 v145 toolset) @@ -216,7 +216,7 @@ jobs: # import-std support. Its opt-in token is CMake-version-specific, so pin # CMake to the version whose token is set in tests/module_cpp20/CMakeLists.txt. - name: Get pinned CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 with: cmakeVersion: 4.3.4 # CMAKE_CXX_FLAGS replaces CMake's MSVC defaults (which include /EHsc), and From e48800f09d6f5e31320371da7ebdda7f7c6e911e Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Wed, 4 Nov 2026 23:42:17 +0100 Subject: [PATCH 8/9] Binary reader and writer: remove checks that invariants make dead (#6001) * Binary reader and writer: remove checks that invariants make dead Branch coverage of the test suite and of 24k mutated inputs showed these conditions as never taken; each is implied by a check made earlier on the same path: - check_bson_document_size: `document_size < 0` (the size was rejected below 5 before it was stored) - get_ubjson_ndarray_size: `second != 'N'` ('N' is rejected as a type marker before `second` can be set) - bjd_type_name() == nullptr for an ndarray dtype: every marker is_valid_optimized_type() accepts for BJData has a name (now an assert) - the early is_bjd_excluded_optimized_type() check in get_ubjson_size_type(): is_valid_optimized_type() rejects the same eight markers with the same error - `input_format == bjdata` next to `is_ndarray`, bit 8 of the type, or a 'B'/'u'/'m'/'M' marker: these only arise for BJData - `first != npos` next to bit 8: the bit is only set after a count - MessagePack writer: the `<= max` halves of the negative int8/16/32 ladder (the ladder is in the `else` of `n >= 0`) No change in behaviour: a build that aborts on each removed side passes the test suite and the fuzz corpora, and a 1.6M-input differential run of the reader against the original headers found no difference. Signed-off-by: Niels Lohmann * Remove is_bjd_excluded_optimized_type(), which has no caller left; fix two comments is_valid_optimized_type() rejects the markers BJData excludes, so the reader no longer needs its own list; the test now checks is_valid_optimized_type() of a BJData reader against the writer's list. The BSON size comment now says that open_bson_document() rejects a size below 5. Signed-off-by: Niels Lohmann * Make the BJData reader in the lookup test const (misc-const-correctness) Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann --- .../nlohmann/detail/input/binary_reader.hpp | 65 +++++----------- .../nlohmann/detail/output/binary_writer.hpp | 11 +-- single_include/nlohmann/json.hpp | 76 ++++++------------- tests/src/unit-bjdata.cpp | 16 ++-- 4 files changed, 57 insertions(+), 111 deletions(-) diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index b793ae097..93f707097 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -344,8 +344,8 @@ class binary_reader nested document could declare a length that disagrees with where its terminator actually falls and quietly hand the bytes in between to the enclosing document. A well-formed document is at least 5 bytes (the prefix - plus the terminator); the equality also rejects those impossible sizes, - since at least 5 bytes are always consumed. + plus the terminator); open_bson_document() rejects a smaller size before + it is stored, so @a document_size is never negative here. @param[in] document_start value of chars_read before the size prefix @param[in] document_size the declared document size @@ -353,7 +353,7 @@ class binary_reader */ bool check_bson_document_size(const std::size_t document_start, const std::int32_t document_size) { - if (JSON_HEDLEY_UNLIKELY(document_size < 0 || static_cast(document_size) != chars_read - document_start)) + if (JSON_HEDLEY_UNLIKELY(static_cast(document_size) != chars_read - document_start)) { return bson_document_size_error(document_size, chars_read - document_start); } @@ -3094,16 +3094,13 @@ class binary_reader { if (size_and_type.second != 0) { - if (size_and_type.second != 'N') + for (std::size_t i = 0; i < size_and_type.first; ++i) { - for (std::size_t i = 0; i < size_and_type.first; ++i) + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_value(dimlen, no_ndarray, size_and_type.second))) { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_value(dimlen, no_ndarray, size_and_type.second))) - { - return false; - } - dim.push_back(dimlen); + return false; } + dim.push_back(dimlen); } } else @@ -3323,11 +3320,9 @@ class binary_reader // follow the documented _ArrayType_, _ArraySize_, _ArrayData_ order if (ndarray_dtype != 0) { + // every optimized type marker is_valid_optimized_type() accepts for BJData has a name const char* type_name = bjd_type_name(ndarray_dtype); - if (JSON_HEDLEY_UNLIKELY(type_name == nullptr)) - { - return unexpected_byte("invalid byte", "type"); - } + JSON_ASSERT(type_name != nullptr); string_t type_key = "_ArrayType_"; string_t type = type_name; // sax->string() takes a reference @@ -3410,12 +3405,6 @@ class binary_reader if (current == '$') { result.second = get(); // must not ignore 'N', because 'N' maybe the type - if (input_format == input_format_t::bjdata - && JSON_HEDLEY_UNLIKELY(is_bjd_excluded_optimized_type(result.second))) - { - return optimized_type_error(); - } - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof("type"))) { return false; @@ -3443,7 +3432,7 @@ class binary_reader // an ndarray was read here only if the flag flipped; when it was // seeded true, get_ubjson_size_value() already rejected the nested // dimension vector - if (input_format == input_format_t::bjdata && is_ndarray && !inside_ndarray) + if (is_ndarray && !inside_ndarray) { result.second |= (1 << 8); // use bit 8 to indicate ndarray, all UBJSON and BJData markers should be ASCII letters } @@ -3453,7 +3442,7 @@ class binary_reader if (current == '#') { const bool is_error = get_ubjson_size_value(result.first, is_ndarray); - if (input_format == input_format_t::bjdata && is_ndarray && !inside_ndarray) + if (is_ndarray && !inside_ndarray) { return last_byte_error(exception_id::unexpected_byte, "ndarray requires both type and size", "size"); } @@ -3627,7 +3616,6 @@ class binary_reader JSON_HEDLEY_NEVER_INLINE bool parse_ubjson_typed_run(std::size_t& remaining, const char_int_type type, std::true_type /*bulk*/) { - const bool bjdata = input_format == input_format_t::bjdata; switch (type) { case 'U': @@ -3641,11 +3629,11 @@ class binary_reader case 'L': return parse_ubjson_typed_run(remaining); case 'u': - return !bjdata || parse_ubjson_typed_run(remaining); + return parse_ubjson_typed_run(remaining); case 'm': - return !bjdata || parse_ubjson_typed_run(remaining); + return parse_ubjson_typed_run(remaining); case 'M': - return !bjdata || parse_ubjson_typed_run(remaining); + return parse_ubjson_typed_run(remaining); case 'd': return parse_ubjson_typed_run(remaining); case 'D': @@ -3726,7 +3714,7 @@ class binary_reader // if bit-8 of size_and_type.second is set to 1, encode bjdata ndarray as an object in JData annotated array format (https://github.com/NeuroJSON/jdata): // {"_ArrayType_" : "typeid", "_ArraySize_" : [n1, n2, ...], "_ArrayData_" : [v1, v2, ...]} - if (input_format == input_format_t::bjdata && size_and_type.first != npos && (size_and_type.second & (1 << 8)) != 0) + if ((size_and_type.second & (1 << 8)) != 0) { size_and_type.second &= ~(static_cast(1) << 8); // use bit 8 to indicate ndarray, here we remove the bit to restore the type marker @@ -3785,7 +3773,7 @@ class binary_reader } // If BJData type marker is 'B' decode as binary - if (input_format == input_format_t::bjdata && size_and_type.first != npos && size_and_type.second == 'B') + if (size_and_type.first != npos && size_and_type.second == 'B') { binary_t result; return get_binary(size_and_type.first, result) && sax->binary(result); @@ -3834,7 +3822,7 @@ class binary_reader } // do not accept ND-array size in objects in BJData - if (input_format == input_format_t::bjdata && size_and_type.first != npos && (size_and_type.second & (1 << 8)) != 0) + if ((size_and_type.second & (1 << 8)) != 0) { return last_byte_error(exception_id::unexpected_byte, "BJData object does not support ND-array size in optimized format", "object"); } @@ -5115,8 +5103,9 @@ class binary_reader JSON_PRIVATE_UNLESS_TESTED: /*! @brief check whether @a marker is a type marker that may follow '$' - UBJSON: Z N T F i U I l L d D H C S [ {; BJData: the numeric markers and C - (BJData's excluded markers are rejected separately). + UBJSON: Z N T F i U I l L d D H C S [ {; BJData: the numeric markers and C. + The markers BJData excludes (F H N S T Z [ {) are the ones + binary_writer::is_bjdata_excluded_type_marker() lists; keep the two in sync. */ bool is_valid_optimized_type(const char_int_type marker) const noexcept { @@ -5151,20 +5140,6 @@ class binary_reader } } - /*! - @brief whether @a marker is excluded from BJData's optimized ND-array types - @return whether @a marker is one of 'F', 'H', 'N', 'S', 'T', 'Z', '[', '{' - - Mirrors binary_writer's @ref binary_writer::is_bjdata_excluded_type_marker - "is_bjdata_excluded_type_marker()`, which encodes the same list the other - way; keep the two in sync. - */ - static constexpr bool is_bjd_excluded_optimized_type(const char_int_type marker) noexcept - { - return marker == '[' || marker == '{' || marker == 'S' || marker == 'H' - || marker == 'T' || marker == 'F' || marker == 'N' || marker == 'Z'; - } - /*! @brief look up the ND-array element type name for a BJData dtype marker @return the type name ("uint8", "int8", ...), or nullptr if @a marker does diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 5bbaa42d2..dda9c3758 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -3126,22 +3126,19 @@ class binary_writer // negative fixnum write_number(static_cast(j.m_data.m_value.number_integer)); } - else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)() && - j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) + else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)()) { // int 8 oa.write_character(to_char_type(0xD0)); write_number(static_cast(j.m_data.m_value.number_integer)); } - else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)() && - j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) + else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)()) { // int 16 oa.write_character(to_char_type(0xD1)); write_number(static_cast(j.m_data.m_value.number_integer)); } - else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)() && - j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) + else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)()) { // int 32 oa.write_character(to_char_type(0xD2)); @@ -4057,7 +4054,7 @@ class binary_writer Containers, strings, high-precision numbers, booleans and null cannot be declared as the single type of an optimized container in BJData; such a container is written unoptimized. The reader rejects them with the same - list (binary_reader::is_bjd_excluded_optimized_type()). + list (binary_reader::is_valid_optimized_type()). */ static constexpr bool is_bjdata_excluded_type_marker(const CharType marker) noexcept { diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 50089514f..eedb9c561 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -18360,8 +18360,8 @@ class binary_reader nested document could declare a length that disagrees with where its terminator actually falls and quietly hand the bytes in between to the enclosing document. A well-formed document is at least 5 bytes (the prefix - plus the terminator); the equality also rejects those impossible sizes, - since at least 5 bytes are always consumed. + plus the terminator); open_bson_document() rejects a smaller size before + it is stored, so @a document_size is never negative here. @param[in] document_start value of chars_read before the size prefix @param[in] document_size the declared document size @@ -18369,7 +18369,7 @@ class binary_reader */ bool check_bson_document_size(const std::size_t document_start, const std::int32_t document_size) { - if (JSON_HEDLEY_UNLIKELY(document_size < 0 || static_cast(document_size) != chars_read - document_start)) + if (JSON_HEDLEY_UNLIKELY(static_cast(document_size) != chars_read - document_start)) { return bson_document_size_error(document_size, chars_read - document_start); } @@ -21110,16 +21110,13 @@ class binary_reader { if (size_and_type.second != 0) { - if (size_and_type.second != 'N') + for (std::size_t i = 0; i < size_and_type.first; ++i) { - for (std::size_t i = 0; i < size_and_type.first; ++i) + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_value(dimlen, no_ndarray, size_and_type.second))) { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_value(dimlen, no_ndarray, size_and_type.second))) - { - return false; - } - dim.push_back(dimlen); + return false; } + dim.push_back(dimlen); } } else @@ -21339,11 +21336,9 @@ class binary_reader // follow the documented _ArrayType_, _ArraySize_, _ArrayData_ order if (ndarray_dtype != 0) { + // every optimized type marker is_valid_optimized_type() accepts for BJData has a name const char* type_name = bjd_type_name(ndarray_dtype); - if (JSON_HEDLEY_UNLIKELY(type_name == nullptr)) - { - return unexpected_byte("invalid byte", "type"); - } + JSON_ASSERT(type_name != nullptr); string_t type_key = "_ArrayType_"; string_t type = type_name; // sax->string() takes a reference @@ -21426,12 +21421,6 @@ class binary_reader if (current == '$') { result.second = get(); // must not ignore 'N', because 'N' maybe the type - if (input_format == input_format_t::bjdata - && JSON_HEDLEY_UNLIKELY(is_bjd_excluded_optimized_type(result.second))) - { - return optimized_type_error(); - } - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof("type"))) { return false; @@ -21459,7 +21448,7 @@ class binary_reader // an ndarray was read here only if the flag flipped; when it was // seeded true, get_ubjson_size_value() already rejected the nested // dimension vector - if (input_format == input_format_t::bjdata && is_ndarray && !inside_ndarray) + if (is_ndarray && !inside_ndarray) { result.second |= (1 << 8); // use bit 8 to indicate ndarray, all UBJSON and BJData markers should be ASCII letters } @@ -21469,7 +21458,7 @@ class binary_reader if (current == '#') { const bool is_error = get_ubjson_size_value(result.first, is_ndarray); - if (input_format == input_format_t::bjdata && is_ndarray && !inside_ndarray) + if (is_ndarray && !inside_ndarray) { return last_byte_error(exception_id::unexpected_byte, "ndarray requires both type and size", "size"); } @@ -21643,7 +21632,6 @@ class binary_reader JSON_HEDLEY_NEVER_INLINE bool parse_ubjson_typed_run(std::size_t& remaining, const char_int_type type, std::true_type /*bulk*/) { - const bool bjdata = input_format == input_format_t::bjdata; switch (type) { case 'U': @@ -21657,11 +21645,11 @@ class binary_reader case 'L': return parse_ubjson_typed_run(remaining); case 'u': - return !bjdata || parse_ubjson_typed_run(remaining); + return parse_ubjson_typed_run(remaining); case 'm': - return !bjdata || parse_ubjson_typed_run(remaining); + return parse_ubjson_typed_run(remaining); case 'M': - return !bjdata || parse_ubjson_typed_run(remaining); + return parse_ubjson_typed_run(remaining); case 'd': return parse_ubjson_typed_run(remaining); case 'D': @@ -21742,7 +21730,7 @@ class binary_reader // if bit-8 of size_and_type.second is set to 1, encode bjdata ndarray as an object in JData annotated array format (https://github.com/NeuroJSON/jdata): // {"_ArrayType_" : "typeid", "_ArraySize_" : [n1, n2, ...], "_ArrayData_" : [v1, v2, ...]} - if (input_format == input_format_t::bjdata && size_and_type.first != npos && (size_and_type.second & (1 << 8)) != 0) + if ((size_and_type.second & (1 << 8)) != 0) { size_and_type.second &= ~(static_cast(1) << 8); // use bit 8 to indicate ndarray, here we remove the bit to restore the type marker @@ -21801,7 +21789,7 @@ class binary_reader } // If BJData type marker is 'B' decode as binary - if (input_format == input_format_t::bjdata && size_and_type.first != npos && size_and_type.second == 'B') + if (size_and_type.first != npos && size_and_type.second == 'B') { binary_t result; return get_binary(size_and_type.first, result) && sax->binary(result); @@ -21850,7 +21838,7 @@ class binary_reader } // do not accept ND-array size in objects in BJData - if (input_format == input_format_t::bjdata && size_and_type.first != npos && (size_and_type.second & (1 << 8)) != 0) + if ((size_and_type.second & (1 << 8)) != 0) { return last_byte_error(exception_id::unexpected_byte, "BJData object does not support ND-array size in optimized format", "object"); } @@ -23131,8 +23119,9 @@ class binary_reader JSON_PRIVATE_UNLESS_TESTED: /*! @brief check whether @a marker is a type marker that may follow '$' - UBJSON: Z N T F i U I l L d D H C S [ {; BJData: the numeric markers and C - (BJData's excluded markers are rejected separately). + UBJSON: Z N T F i U I l L d D H C S [ {; BJData: the numeric markers and C. + The markers BJData excludes (F H N S T Z [ {) are the ones + binary_writer::is_bjdata_excluded_type_marker() lists; keep the two in sync. */ bool is_valid_optimized_type(const char_int_type marker) const noexcept { @@ -23167,20 +23156,6 @@ class binary_reader } } - /*! - @brief whether @a marker is excluded from BJData's optimized ND-array types - @return whether @a marker is one of 'F', 'H', 'N', 'S', 'T', 'Z', '[', '{' - - Mirrors binary_writer's @ref binary_writer::is_bjdata_excluded_type_marker - "is_bjdata_excluded_type_marker()`, which encodes the same list the other - way; keep the two in sync. - */ - static constexpr bool is_bjd_excluded_optimized_type(const char_int_type marker) noexcept - { - return marker == '[' || marker == '{' || marker == 'S' || marker == 'H' - || marker == 'T' || marker == 'F' || marker == 'N' || marker == 'Z'; - } - /*! @brief look up the ND-array element type name for a BJData dtype marker @return the type name ("uint8", "int8", ...), or nullptr if @a marker does @@ -31144,22 +31119,19 @@ class binary_writer // negative fixnum write_number(static_cast(j.m_data.m_value.number_integer)); } - else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)() && - j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) + else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)()) { // int 8 oa.write_character(to_char_type(0xD0)); write_number(static_cast(j.m_data.m_value.number_integer)); } - else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)() && - j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) + else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)()) { // int 16 oa.write_character(to_char_type(0xD1)); write_number(static_cast(j.m_data.m_value.number_integer)); } - else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)() && - j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) + else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)()) { // int 32 oa.write_character(to_char_type(0xD2)); @@ -32075,7 +32047,7 @@ class binary_writer Containers, strings, high-precision numbers, booleans and null cannot be declared as the single type of an optimized container in BJData; such a container is written unoptimized. The reader rejects them with the same - list (binary_reader::is_bjd_excluded_optimized_type()). + list (binary_reader::is_valid_optimized_type()). */ static constexpr bool is_bjdata_excluded_type_marker(const CharType marker) noexcept { diff --git a/tests/src/unit-bjdata.cpp b/tests/src/unit-bjdata.cpp index ecc79959d..dd5831dbd 100644 --- a/tests/src/unit-bjdata.cpp +++ b/tests/src/unit-bjdata.cpp @@ -78,23 +78,25 @@ TEST_CASE("BJData") { SECTION("binary_reader BJData lookup tables") { - // both lookups are static member functions + // bjd_type_name() is static; is_valid_optimized_type() needs a BJData reader std::vector const data; using reader_t = nlohmann::detail::binary_reader; - // the excluded optimized-type markers must match binary_writer's - // is_bjdata_excluded_type_marker(), which encodes the same 8 markers + // the markers BJData excludes as optimized types must match + // binary_writer's is_bjdata_excluded_type_marker(), which encodes the + // same 8 markers + const reader_t reader(nlohmann::detail::input_adapter(data), json::input_format_t::bjdata); for (const char marker : - {'[', '{', 'S', 'H', 'T', 'F', 'N', 'Z' + {'[', '{', 'S', 'H', 'T', 'F', 'N', 'Z', 'x' }) { - CHECK(reader_t::is_bjd_excluded_optimized_type(static_cast(marker))); + CHECK(!reader.is_valid_optimized_type(static_cast(marker))); } for (const char marker : - {'U', 'i', 'u', 'I', 'm', 'l', 'M', 'L', 'd', 'D', 'C', 'B', 'x' + {'U', 'i', 'u', 'I', 'm', 'l', 'M', 'L', 'd', 'D', 'C', 'B', 'h' }) { - CHECK(!reader_t::is_bjd_excluded_optimized_type(static_cast(marker))); + CHECK(reader.is_valid_optimized_type(static_cast(marker))); } // every dtype marker must round-trip to its ND-array type name From b54267d2a74c085ff2b05b01e369ecb7dd70cec6 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Thu, 5 Nov 2026 16:04:39 +0100 Subject: [PATCH 9/9] Docs: compile times of JSON_HAS_STD_FORMAT=0 and JSON_HAS_RANGES=0 (#6025) Add sections on switching off std::format and ranges support to the compile-times page, with measurements (Apple clang, GCC 16, -std=c++20), the functionality that is lost, and chart rows (issue #5294). Signed-off-by: Niels Lohmann --- docs/mkdocs/docs/images/compile_times.svg | 56 ++++++----- docs/mkdocs/docs/integration/compile_times.md | 92 ++++++++++++++++++- docs/mkdocs/scripts/figures/charts.py | 2 + 3 files changed, 126 insertions(+), 24 deletions(-) diff --git a/docs/mkdocs/docs/images/compile_times.svg b/docs/mkdocs/docs/images/compile_times.svg index e8e904a60..302817e1c 100644 --- a/docs/mkdocs/docs/images/compile_times.svg +++ b/docs/mkdocs/docs/images/compile_times.svg @@ -1,21 +1,21 @@ - + Change in compile time against the default configuration at -O0, Apple clang and G​CC 16 -Grouped horizontal bars, negative is faster. json_fwd.hpp instead of json.hpp on a header-only translation unit: Apple clang -62 %, G​CC 16 -82 %; extern template on a parsing translation unit: Apple clang -7 %, G​CC 16 -21 %; JSON_NO_AUTOMATIC_UDLS on a model translation unit: Apple clang -11 %, G​CC 16 -11 %; JSON_NO_AUTOMATIC_UDLS on a parsing translation unit: Apple clang -1 %, G​CC 16 -2 %; JSON_HAS_FILESYSTEM=0 + JSON_NO_IO on a model translation unit: Apple clang -8 %, G​CC 16 -11 %; JSON_HAS_FILESYSTEM=0 on a model translation unit: Apple clang -5 %, G​CC 16 -9 %; JSON_NO_IO on a model translation unit: Apple clang -2 %, G​CC 16 -2 %; JSON_USE_GLOBAL_UDLS=0 on a model translation unit: Apple clang 0 %, G​CC 16 -1 %. +Grouped horizontal bars, negative is faster. json_fwd.hpp instead of json.hpp on a header-only translation unit: Apple clang -62 %, G​CC 16 -82 %; extern template on a parsing translation unit: Apple clang -7 %, G​CC 16 -21 %; JSON_NO_AUTOMATIC_UDLS on a model translation unit: Apple clang -11 %, G​CC 16 -11 %; JSON_NO_AUTOMATIC_UDLS on a parsing translation unit: Apple clang -1 %, G​CC 16 -2 %; JSON_HAS_FILESYSTEM=0 + JSON_NO_IO on a model translation unit: Apple clang -8 %, G​CC 16 -11 %; JSON_HAS_FILESYSTEM=0 on a model translation unit: Apple clang -5 %, G​CC 16 -9 %; JSON_HAS_STD_FORMAT=0 on a model translation unit: Apple clang -1 %, G​CC 16 -4 %; JSON_HAS_RANGES=0 on a model translation unit: Apple clang -4 %, G​CC 16 -1 %; JSON_NO_IO on a model translation unit: Apple clang -2 %, G​CC 16 -2 %; JSON_USE_GLOBAL_UDLS=0 on a model translation unit: Apple clang 0 %, G​CC 16 -1 %. Apple clang G​CC 16 - -−80 % - -−60 % - -−40 % - -−20 % -0 % -← faster -slower → + +−80 % + +−60 % + +−40 % + +−20 % +0 % +← faster +slower → json_fwd.hpp instead of json.hpp · header-only TU json_fwd.hpp instead of json.hpp, header-only TU (Apple clang): −62 % −62 % @@ -46,15 +46,25 @@ −5 % JSON_HAS_FILESYSTEM=0, model TU (G​CC 16): −9 % −9 % -JSON_NO_IO · model TU* -JSON_NO_IO, model TU (Apple clang): −2 % -−2 % -JSON_NO_IO, model TU (G​CC 16): −2 % -−2 % -JSON_USE_GLOBAL_UDLS=0 · model TU* -JSON_USE_GLOBAL_UDLS=0, model TU (Apple clang): 0 % -0 % -JSON_USE_GLOBAL_UDLS=0, model TU (G​CC 16): −1 % +JSON_HAS_STD_FORMAT=0 · model TU +JSON_HAS_STD_FORMAT=0, model TU (Apple clang): −1 % +−1 % +JSON_HAS_STD_FORMAT=0, model TU (G​CC 16): −4 % +−4 % +JSON_HAS_RANGES=0 · model TU +JSON_HAS_RANGES=0, model TU (Apple clang): −4 % +−4 % +JSON_HAS_RANGES=0, model TU (G​CC 16): −1 % −1 % - +JSON_NO_IO · model TU* +JSON_NO_IO, model TU (Apple clang): −2 % +−2 % +JSON_NO_IO, model TU (G​CC 16): −2 % +−2 % +JSON_USE_GLOBAL_UDLS=0 · model TU* +JSON_USE_GLOBAL_UDLS=0, model TU (Apple clang): 0 % +0 % +JSON_USE_GLOBAL_UDLS=0, model TU (G​CC 16): −1 % +−1 % + diff --git a/docs/mkdocs/docs/integration/compile_times.md b/docs/mkdocs/docs/integration/compile_times.md index 8798931cb..699b21f05 100644 --- a/docs/mkdocs/docs/integration/compile_times.md +++ b/docs/mkdocs/docs/integration/compile_times.md @@ -11,6 +11,11 @@ that cost, ordered by how much they typically save. 16.2.0 on macOS (Apple M1 Max). All configurations were run interleaved, in alternating order. They show the order of magnitude to expect; measure your own code before and after a change. + The two sections on `std::format` and ranges apply to C++20 only and were measured with `-std=c++20`, so their + baselines are slower than the others. Their differences are mostly within the run-to-run noise of a few percent: + two complete runs of the same configuration differed by up to 10% in the median. The tables show the later run (one + outlier cell was repeated). + The translation units are listed below. They replace the ones used for earlier versions of this page, whose sources were not published, so the numbers differ from earlier versions. @@ -66,7 +71,8 @@ that cost, ordered by how much they typically save. --8<-- "images/compile_times.svg"

Change in compile time at `-O0` against the default configuration of the same -translation unit, taken from the tables below (medians of nine runs, see above). * computed from the milliseconds in +translation unit, taken from the tables below (medians of nine runs, see above; the `std::format` and ranges rows are +measured with `-std=c++20`). * computed from the milliseconds in [Options without effect on compile times](#options-without-effect-on-compile-times).

## Include `json_fwd.hpp` in headers @@ -214,6 +220,88 @@ from and to `std::string`; whether that fallback exists depends on the platform' (`std::wstring` on Windows), so check code that converts paths. `JSON_NO_IO` additionally removes all stream functions (see [Options without effect](#options-without-effect-on-compile-times)). +## Switch off `std::format` support + +With C++20 and a standard library that provides ``, the library includes it to provide a +[`std::formatter`](../api/basic_json/std_formatter.md) specialization (see +[`JSON_HAS_STD_FORMAT`](../api/macros/json_has_std_format.md)). If you do not format JSON values with `std::format`, +switch the detection off for the whole project: + +```cmake +target_compile_definitions(my_target PRIVATE JSON_HAS_STD_FORMAT=0) +``` + +| Compiler | Translation unit | Default (`-O0` / `-O2`) | `std::format` off (`-O0` / `-O2`) | Change | +|-------------|------------------|------------------------:|----------------------------------:|-----------:| +| Apple clang | model | 1283 ms / 1544 ms | 1269 ms / 1539 ms | −1% / 0% | +| GCC 16 | model | 2026 ms / 2482 ms | 1944 ms / 2416 ms | −4% / −3% | +| Apple clang | parsing | 1350 ms / 2278 ms | 1347 ms / 2300 ms | 0% / +1% | +| GCC 16 | parsing | 2715 ms / 4351 ms | 2644 ms / 4287 ms | −3% / −1% | + +

Compile time with `JSON_HAS_STD_FORMAT=0` against the default configuration, both with `-std=c++20`, at `-O0` and `-O2`.

+ +The saving is small, and only GCC shows it: libstdc++'s `` is an additional header that nothing else in the +translation unit includes, so the macro removes it (and the headers it pulls in). With libc++ (Apple clang), other +standard headers include `` anyway, so the macro only removes the library's own formatter, and the difference is +within the noise. + +The trade-off: the `std::formatter` specialization is removed, so `#!cpp std::format("{}", j)`, +and `#!cpp std::format_to` with a JSON value no longer compile (in a test with both compilers on +macOS, the error was that `std::formatter` must be specialized for the type). Use [`dump()`](../api/basic_json/dump.md), +or the stream operators, instead. + +## Switch off ranges support + +With C++20 and a standard library that provides ``, the library includes it to declare the +[`items()`](../api/basic_json/items.md) proxy a *borrowed range*, and uses the `std::ranges` concepts and algorithms for +construction from range views and for iterator input (see [`JSON_HAS_RANGES`](../api/macros/json_has_ranges.md)). If you +do not need these, switch the detection off for the whole project: + +```cmake +target_compile_definitions(my_target PRIVATE JSON_HAS_RANGES=0) +``` + +| Compiler | Translation unit | Default (`-O0` / `-O2`) | Ranges off (`-O0` / `-O2`) | Change | +|-------------|------------------|------------------------:|---------------------------:|-----------:| +| Apple clang | model | 1283 ms / 1544 ms | 1237 ms / 1495 ms | −4% / −3% | +| GCC 16 | model | 2026 ms / 2482 ms | 2004 ms / 2464 ms | −1% / −1% | +| Apple clang | parsing | 1350 ms / 2278 ms | 1311 ms / 2284 ms | −3% / 0% | +| GCC 16 | parsing | 2715 ms / 4351 ms | 2705 ms / 4427 ms | 0% / +2% | + +

Compile time with `JSON_HAS_RANGES=0` against the default configuration, both with `-std=c++20`, at `-O0` and `-O2`.

+ +The saving is small as well, and with GCC it is not distinguishable from the noise: its `` includes ``, +so with `std::format` support left on, part of the ranges machinery is still parsed. +Both macros together save more than either alone: + +```cmake +target_compile_definitions(my_target PRIVATE JSON_HAS_STD_FORMAT=0 JSON_HAS_RANGES=0) +``` + +| Compiler | Translation unit | Default (`-O0` / `-O2`) | Both off (`-O0` / `-O2`) | Change | +|-------------|------------------|------------------------:|-------------------------:|-----------:| +| Apple clang | model | 1283 ms / 1544 ms | 1223 ms / 1504 ms | −5% / −3% | +| GCC 16 | model | 2026 ms / 2482 ms | 1914 ms / 2378 ms | −6% / −4% | +| Apple clang | parsing | 1350 ms / 2278 ms | 1311 ms / 2271 ms | −3% / 0% | +| GCC 16 | parsing | 2715 ms / 4351 ms | 2635 ms / 4260 ms | −3% / −2% | + +

Compile time with `JSON_HAS_STD_FORMAT=0` and `JSON_HAS_RANGES=0` against the default configuration, both with `-std=c++20`, at `-O0` and `-O2`.

+ +The trade-offs of `JSON_HAS_RANGES=0` are larger than those of the other options on this page: + +- `items()` is no longer a borrowed range. On a temporary, `#!cpp std::ranges::begin(make_json().items())` no longer + compiles, and `std::ranges` algorithms such as `#!cpp std::ranges::find_if(make_json().items(), pred)` return + `std::ranges::dangling` instead of an iterator, so code that uses the result no longer compiles. Calling them on + `#!cpp j.items()` for a named `j` is not affected. +- [`JSON_HAS_RANGE_VIEW_CONVERSION`](../api/macros/json_has_range_view_conversion.md) becomes `0`, so a JSON array can no + longer be constructed from a `std::ranges::view` such as the result of `std::views::filter`. The assignment + `#!cpp json j = v | std::views::filter(pred);` no longer compiles. +- Parsing from an iterator pair loses the C++20 fast path for contiguous iterators that are not pointers (for + example those of `std::vector`) and falls back to byte-at-a-time scanning. The result is the same; parsing may be + slower (not measured here). + +The first two were checked with both compilers on macOS; the third is read from the library's source. + ## Use C++20 modules With a toolchain that supports named modules, `import nlohmann.json;` compiles the library once into a module and @@ -343,6 +431,8 @@ and −2% / 0% (GCC). To get a saving, combine it with - [`JSON_HAS_FILESYSTEM`](../api/macros/json_has_filesystem.md) - control the conversions from and to `std::filesystem::path` +- [`JSON_HAS_STD_FORMAT`](../api/macros/json_has_std_format.md) - control the `std::formatter` specialization +- [`JSON_HAS_RANGES`](../api/macros/json_has_ranges.md) - control the use of `std::ranges` - [`JSON_NO_AUTOMATIC_UDLS`](../api/macros/json_no_automatic_udls.md) - do not include the user-defined string literals automatically - [Modules](../features/modules.md) - C++20 module support diff --git a/docs/mkdocs/scripts/figures/charts.py b/docs/mkdocs/scripts/figures/charts.py index 6f1c4dc3e..d0440977c 100644 --- a/docs/mkdocs/scripts/figures/charts.py +++ b/docs/mkdocs/scripts/figures/charts.py @@ -73,6 +73,8 @@ COMPILE = [ ("JSON_NO_AUTOMATIC_UDLS", "parsing TU", -1, -2, False), ("JSON_HAS_FILESYSTEM=0 + JSON_NO_IO", "model TU", -8, -11, False), ("JSON_HAS_FILESYSTEM=0", "model TU", -5, -9, False), + ("JSON_HAS_STD_FORMAT=0", "model TU", -1, -4, False), + ("JSON_HAS_RANGES=0", "model TU", -4, -1, False), # computed from the ms in "Options without effect on compile times": 1034/1051, 1756/1801, 1055/1051, 1783/1801 ("JSON_NO_IO", "model TU", round((1034 / 1051 - 1) * 100), round((1756 / 1801 - 1) * 100), True), ("JSON_USE_GLOBAL_UDLS=0", "model TU", round((1055 / 1051 - 1) * 100), round((1783 / 1801 - 1) * 100), True),